largeFileHandler

base/herr.h

@@ -43,6 +43,8 @@
     \
     F(1030, OPEN_FILE,      "Open file failed") \
     F(1031, SAVE_FILE,      "Save file failed") \
+    F(1032, READ_FILE,      "Read file failed") \
+    F(1033, WRITE_FILE,     "Write file failed")\
     \
     F(1100, TASK_TIMEOUT,       "Task timeout")     \
     F(1101, TASK_QUEUE_FULL,    "Task queue full")  \

examples/httpd/handler.h

@@ -1,10 +1,14 @@
 #ifndef HV_HTTPD_HANDLER_H
 #define HV_HTTPD_HANDLER_H
 
-#include "HttpMessage.h"
-#include "HttpResponseWriter.h"
+#include <future> // import std::async
+
+#include "hbase.h"
 #include "htime.h"
+#include "hfile.h"
+#include "hstring.h"
 #include "EventLoop.h" // import setTimeout, setInterval
+#include "HttpService.h"
 
 class Handler {
 public:
@@ -48,6 +52,55 @@ public:
         return 0;
     }
 
+    static int errorHandler(const HttpContextPtr& ctx) {
+        int error_code = ctx->response->status_code;
+        return response_status(ctx->response.get(), error_code);
+    }
+
+    static int largeFileHandler(const HttpContextPtr& ctx) {
+        std::async([ctx](){
+            ctx->writer->Begin();
+            std::string filepath = ctx->service->document_root + ctx->request->Path();
+            HFile file;
+            if (file.open(filepath.c_str(), "rb") != 0) {
+                ctx->writer->WriteStatus(HTTP_STATUS_NOT_FOUND);
+                ctx->writer->WriteHeader("Content-Type", "text/html");
+                ctx->writer->WriteBody("<center><h1>404 Not Found</h1></center>");
+                ctx->writer->End();
+                return;
+            }
+            http_content_type content_type = CONTENT_TYPE_NONE;
+            const char* suffix = hv_suffixname(filepath.c_str());
+            if (suffix) {
+                content_type = http_content_type_enum_by_suffix(suffix);
+            }
+            if (content_type == CONTENT_TYPE_NONE || content_type == CONTENT_TYPE_UNDEFINED) {
+                content_type = APPLICATION_OCTET_STREAM;
+            }
+            size_t filesize = file.size();
+            ctx->writer->WriteHeader("Content-Type", http_content_type_str(content_type));
+            ctx->writer->WriteHeader("Content-Length", filesize);
+            ctx->writer->EndHeaders();
+
+            char* buf = NULL;
+            int len = 4096; // 4K
+            SAFE_ALLOC(buf, len);
+            size_t total_readbytes = 0;
+            while (total_readbytes < filesize) {
+                size_t readbytes = file.read(buf, len);
+                if (readbytes <= 0) {
+                    ctx->writer->close();
+                    break;
+                }
+                ctx->writer->WriteBody(buf, readbytes);
+                total_readbytes += readbytes;
+            }
+            ctx->writer->End();
+            SAFE_FREE(buf);
+        });
+        return 0;
+    }
+
     static int sleep(HttpRequest* req, HttpResponse* resp) {
         resp->Set("start_ms", gettimeofday_ms());
         std::string strTime = req->GetParam("t");

examples/httpd/router.h

@@ -14,6 +14,8 @@ public:
         // preprocessor => Handler => postprocessor
         router.preprocessor = Handler::preprocessor;
         router.postprocessor = Handler::postprocessor;
+        router.largeFileHandler = Handler::largeFileHandler;
+        // router.errorHandler = Handler::errorHandler;
 
         // curl -v http://ip:port/ping
         router.GET("/ping", [](HttpRequest* req, HttpResponse* resp) {

http/server/FileCache.cpp

@@ -1,5 +1,6 @@
 #include "FileCache.h"
 
+#include "herr.h"
 #include "hscope.h"
 #include "htime.h"
 #include "hlog.h"
@@ -9,7 +10,7 @@
 
 #define ETAG_FMT    "\"%zx-%zx\""
 
-file_cache_ptr FileCache::Open(const char* filepath, bool need_read, void* ctx) {
+file_cache_ptr FileCache::Open(const char* filepath, OpenParam* param) {
     std::lock_guard<std::mutex> locker(mutex_);
     file_cache_ptr fc = Get(filepath);
     bool modified = false;
@@ -20,19 +21,20 @@ file_cache_ptr FileCache::Open(const char* filepath, bool need_read, void* ctx)
             fc->stat_time = now;
             fc->stat_cnt++;
         }
-        if (need_read) {
+        if (param->need_read) {
             if (!modified && fc->is_complete()) {
-                need_read = false;
+                param->need_read = false;
             }
         }
     }
-    if (fc == NULL || modified || need_read) {
+    if (fc == NULL || modified || param->need_read) {
         int flags = O_RDONLY;
 #ifdef O_BINARY
         flags |= O_BINARY;
 #endif
         int fd = open(filepath, flags);
         if (fd < 0) {
+            param->error = ERR_OPEN_FILE;
             return NULL;
         }
         defer(close(fd);)
@@ -51,20 +53,23 @@ file_cache_ptr FileCache::Open(const char* filepath, bool need_read, void* ctx)
                 cached_files[filepath] = fc;
             }
             else {
+                param->error = ERR_MISMATCH;
                 return NULL;
             }
         }
         if (S_ISREG(fc->st.st_mode)) {
+            param->filesize = fc->st.st_size;
             // FILE
-            if (need_read) {
-                if (fc->st.st_size > FILE_CACHE_MAX_SIZE) {
-                    hlogw("Too large file: %s", filepath);
+            if (param->need_read) {
+                if (fc->st.st_size > param->max_read) {
+                    param->error = ERR_OVER_LIMIT;
                     return NULL;
                 }
                 fc->resize_buf(fc->st.st_size);
                 int nread = read(fd, fc->filebuf.base, fc->filebuf.len);
                 if (nread != fc->filebuf.len) {
                     hloge("Failed to read file: %s", filepath);
+                    param->error = ERR_READ_FILE;
                     return NULL;
                 }
             }
@@ -83,7 +88,7 @@ file_cache_ptr FileCache::Open(const char* filepath, bool need_read, void* ctx)
         else if (S_ISDIR(fc->st.st_mode)) {
             // DIR
             std::string page;
-            make_index_of_page(filepath, page, (const char*)ctx);
+            make_index_of_page(filepath, page, param->path);
             fc->resize_buf(page.size());
             memcpy(fc->filebuf.base, page.c_str(), page.size());
             fc->content_type = "text/html; charset=utf-8";

http/server/FileCache.h

@@ -10,7 +10,7 @@
 #include "hstring.h"
 
 #define HTTP_HEADER_MAX_LENGTH      1024        // 1K
-#define FILE_CACHE_MAX_SIZE         (1 << 30)   // 1G
+#define FILE_CACHE_MAX_SIZE         (1 << 26)   // 64M
 
 typedef struct file_cache_s {
     std::string filepath;
@@ -74,7 +74,22 @@ public:
         file_expired_time  = DEFAULT_FILE_EXPIRED_TIME;
     }
 
-    file_cache_ptr Open(const char* filepath, bool need_read = true, void* ctx = NULL);
+    struct OpenParam {
+        bool need_read;
+        int  max_read;
+        const char* path;
+        size_t  filesize;
+        int  error;
+
+        OpenParam() {
+            need_read = true;
+            max_read = FILE_CACHE_MAX_SIZE;
+            path = "/";
+            filesize = 0;
+            error = 0;
+        }
+    };
+    file_cache_ptr Open(const char* filepath, OpenParam* param);
     bool Close(const char* filepath);
     bool Close(const file_cache_ptr& fc);
     void RemoveExpiredFileCache();

http/server/HttpHandler.cpp

@@ -1,15 +1,13 @@
 #include "HttpHandler.h"
 
 #include "hbase.h"
+#include "herr.h"
+#include "hlog.h"
 #include "http_page.h"
 
 int HttpHandler::HandleHttpRequest() {
-    // preprocessor -> api -> web -> postprocessor
-
+    // preprocessor -> processor -> postprocessor
     int status_code = HTTP_STATUS_OK;
-    http_sync_handler sync_handler = NULL;
-    http_async_handler async_handler = NULL;
-
     HttpRequest* pReq = req.get();
     HttpResponse* pResp = resp.get();
 
@@ -18,77 +16,95 @@ int HttpHandler::HandleHttpRequest() {
     pReq->client_addr.port = port;
     pReq->Host();
     pReq->ParseUrl();
+    // pReq->ParseBody();
 
 preprocessor:
     state = HANDLE_BEGIN;
     if (service->preprocessor) {
         status_code = service->preprocessor(pReq, pResp);
         if (status_code != 0) {
-            goto make_http_status_page;
+            goto postprocessor;
         }
     }
 
+processor:
+    if (service->processor) {
+        status_code = customHttpHandler(service->processor);
+    } else {
+        status_code = defaultRequestHandler();
+    }
+
+postprocessor:
+    if (status_code >= 100 && status_code < 600) {
+        pResp->status_code = (http_status)status_code;
+    }
+    if (pResp->status_code >= 400 && pResp->body.size() == 0 && pReq->method != HTTP_HEAD) {
+        if (service->errorHandler) {
+            customHttpHandler(service->errorHandler);
+        } else {
+            defaultErrorHandler();
+        }
+    }
+    if (fc) {
+        pResp->content = fc->filebuf.base;
+        pResp->content_length = fc->filebuf.len;
+        pResp->headers["Content-Type"] = fc->content_type;
+        pResp->headers["Last-Modified"] = fc->last_modified;
+        pResp->headers["Etag"] = fc->etag;
+    }
+    if (service->postprocessor) {
+        service->postprocessor(pReq, pResp);
+    }
+
+    if (status_code == 0) {
+        state = HANDLE_CONTINUE;
+    } else {
+        state = HANDLE_END;
+        parser->SubmitResponse(resp.get());
+    }
+    return status_code;
+}
+
+int HttpHandler::customHttpHandler(http_handler& fn) {
+    HttpContextPtr ctx(new HttpContext);
+    ctx->service = service;
+    ctx->request = req;
+    ctx->response = resp;
+    ctx->writer = writer;
+    return fn(ctx);
+}
+
+int HttpHandler::defaultRequestHandler() {
+    int status_code = HTTP_STATUS_OK;
+    http_sync_handler sync_handler = NULL;
+    http_async_handler async_handler = NULL;
+
     if (service->api_handlers.size() != 0) {
-        service->GetApi(pReq, &sync_handler, &async_handler);
+        service->GetApi(req.get(), &sync_handler, &async_handler);
     }
 
     if (sync_handler) {
-        // sync api service
-        status_code = sync_handler(pReq, pResp);
+        // sync api handler
+        status_code = sync_handler(req.get(), resp.get());
         if (status_code != 0) {
-            goto make_http_status_page;
+            return status_code;
         }
     }
     else if (async_handler) {
-        // async api service
+        // async api handler
         async_handler(req, writer);
         status_code = 0;
     }
-    else if (service->document_root.size() != 0 &&
-            (pReq->method == HTTP_GET || pReq->method == HTTP_HEAD)) {
-        // file service
-        status_code = 200;
-        std::string path = pReq->Path();
-        const char* req_path = path.c_str();
-        // path safe check
-        if (*req_path != '/' || strstr(req_path, "/../")) {
-            pResp->status_code = HTTP_STATUS_BAD_REQUEST;
-            goto make_http_status_page;
-        }
-        std::string filepath = service->document_root;
-        filepath += req_path;
-        if (req_path[1] == '\0') {
-            filepath += service->home_page;
+    else if (req->method == HTTP_GET || req->method == HTTP_HEAD) {
+        // static handler
+        if (service->staticHandler) {
+            customHttpHandler(service->staticHandler);
         }
-        bool is_dir = filepath.c_str()[filepath.size()-1] == '/';
-        bool is_index_of = false;
-        if (service->index_of.size() != 0 && strstartswith(req_path, service->index_of.c_str())) {
-            is_index_of = true;
-        }
-        if (!is_dir || is_index_of) {
-            bool need_read = pReq->method == HTTP_HEAD ? false : true;
-            fc = files->Open(filepath.c_str(), need_read, (void*)req_path);
-        }
-        if (fc == NULL) {
-            // Not Found
-            status_code = HTTP_STATUS_NOT_FOUND;
+        else if (service->document_root.size() != 0) {
+            status_code = defaultStaticHandler();
         }
         else {
-            // Not Modified
-            auto iter = pReq->headers.find("if-not-match");
-            if (iter != pReq->headers.end() &&
-                strcmp(iter->second.c_str(), fc->etag) == 0) {
-                status_code = HTTP_STATUS_NOT_MODIFIED;
-                fc = NULL;
-            }
-            else {
-                iter = pReq->headers.find("if-modified-since");
-                if (iter != pReq->headers.end() &&
-                    strcmp(iter->second.c_str(), fc->last_modified) == 0) {
-                    status_code = HTTP_STATUS_NOT_MODIFIED;
-                    fc = NULL;
-                }
-            }
+            status_code = HTTP_STATUS_NOT_FOUND;
         }
     }
     else {
@@ -96,45 +112,79 @@ preprocessor:
         status_code = HTTP_STATUS_NOT_IMPLEMENTED;
     }
 
-make_http_status_page:
-    if (status_code >= 100 && status_code < 600) {
-        pResp->status_code = (http_status)status_code;
+    return status_code;
+}
+
+int HttpHandler::defaultStaticHandler() {
+    // file service
+    int status_code = HTTP_STATUS_OK;
+    std::string path = req->Path();
+    const char* req_path = path.c_str();
+    // path safe check
+    if (req_path[0] != '/' || strstr(req_path, "/../")) {
+        return HTTP_STATUS_BAD_REQUEST;
     }
-    if (pResp->status_code >= 400 && pResp->body.size() == 0 && pReq->method != HTTP_HEAD) {
-        // error page
-        if (service->error_page.size() != 0) {
-            std::string filepath = service->document_root;
-            filepath += '/';
-            filepath += service->error_page;
-            fc = files->Open(filepath.c_str(), true, NULL);
-        }
-        // status page
-        if (fc == NULL && pResp->body.size() == 0) {
-            pResp->content_type = TEXT_HTML;
-            make_http_status_page(pResp->status_code, pResp->body);
+    std::string filepath = service->document_root + path;
+    if (req_path[1] == '\0') {
+        filepath += service->home_page;
+    }
+    bool is_dir = filepath.c_str()[filepath.size()-1] == '/';
+    bool is_index_of = false;
+    if (service->index_of.size() != 0 && strstartswith(req_path, service->index_of.c_str())) {
+        is_index_of = true;
+    }
+    if (!is_dir || is_index_of) {
+        FileCache::OpenParam param;
+        param.need_read = req->method == HTTP_HEAD ? false : true;
+        param.path = req_path;
+        fc = files->Open(filepath.c_str(), &param);
+        if (fc == NULL) {
+            status_code = HTTP_STATUS_NOT_FOUND;
+            if (param.error == ERR_OVER_LIMIT) {
+                if (service->largeFileHandler) {
+                    status_code = customHttpHandler(service->largeFileHandler);
+                }
+            }
         }
+    } else {
+        status_code = HTTP_STATUS_NOT_FOUND;
     }
 
     if (fc) {
-        pResp->content = fc->filebuf.base;
-        pResp->content_length = fc->filebuf.len;
-        pResp->headers["Content-Type"] = fc->content_type;
-        pResp->headers["Last-Modified"] = fc->last_modified;
-        pResp->headers["Etag"] = fc->etag;
+        // Not Modified
+        auto iter = req->headers.find("if-not-match");
+        if (iter != req->headers.end() &&
+            strcmp(iter->second.c_str(), fc->etag) == 0) {
+            status_code = HTTP_STATUS_NOT_MODIFIED;
+            fc = NULL;
+        }
+        else {
+            iter = req->headers.find("if-modified-since");
+            if (iter != req->headers.end() &&
+                strcmp(iter->second.c_str(), fc->last_modified) == 0) {
+                status_code = HTTP_STATUS_NOT_MODIFIED;
+                fc = NULL;
+            }
+        }
     }
+    return status_code;
+}
 
-postprocessor:
-    if (service->postprocessor) {
-        service->postprocessor(pReq, pResp);
+int HttpHandler::defaultErrorHandler() {
+    // error page
+    if (service->error_page.size() != 0) {
+        std::string filepath = service->document_root;
+        filepath += '/';
+        filepath += service->error_page;
+        FileCache::OpenParam param;
+        fc = files->Open(filepath.c_str(), &param);
     }
-
-    if (status_code == 0) {
-        state = HANDLE_CONTINUE;
-    } else {
-        state = HANDLE_END;
-        parser->SubmitResponse(pResp);
+    // status page
+    if (fc == NULL && resp->body.size() == 0) {
+        resp->content_type = TEXT_HTML;
+        make_http_status_page(resp->status_code, resp->body);
     }
-    return status_code;
+    return 0;
 }
 
 int HttpHandler::FeedRecvData(const char* data, size_t len) {

http/server/HttpHandler.h

@@ -8,8 +8,6 @@
 #include "WebSocketServer.h"
 #include "WebSocketParser.h"
 
-#include "hlog.h"
-
 class WebSocketHandler {
 public:
     WebSocketChannelPtr         channel;
@@ -26,17 +24,6 @@ public:
 
     void onopen() {
         channel->status = hv::SocketChannel::CONNECTED;
-        /*
-        channel->onread = [this](hv::Buffer* buf) {
-            const char* data = (const char*)buf->data();
-            int size= buf->size();
-            int nfeed = parser->FeedRecvData(data, size);
-            if (nfeed != size) {
-                hloge("websocket parse error!");
-                channel->close();
-            }
-        };
-        */
     }
 
     void onclose() {
@@ -164,6 +151,12 @@ public:
             ws_service->onmessage(ws->channel, msg);
         }
     }
+
+private:
+    int defaultRequestHandler();
+    int defaultStaticHandler();
+    int defaultErrorHandler();
+    int customHttpHandler(http_handler& fn);
 };
 
 #endif // HV_HTTP_HANDLER_H_

http/server/HttpResponseWriter.h

@@ -42,6 +42,12 @@ public:
         return 0;
     }
 
+    template<typename T>
+    int WriteHeader(const char* key, T num) {
+        response->headers[key] = hv::to_string(num);
+        return 0;
+    }
+
     int EndHeaders(const char* key = NULL, const char* value = NULL) {
         if (state != SEND_BEGIN) return -1;
         if (key && value) {

http/server/HttpService.h

@@ -15,6 +15,7 @@
 #define DEFAULT_DOCUMENT_ROOT   "/var/www/html"
 #define DEFAULT_HOME_PAGE       "index.html"
 #define DEFAULT_ERROR_PAGE      "error.html"
+#define DEFAULT_INDEXOF_DIR     "/downloads/"
 
 /*
  * @param[in]  req:  parsed structured http request
@@ -43,27 +44,54 @@ typedef std::list<http_method_handler>                                  http_met
 // path => http_method_handlers
 typedef std::map<std::string, std::shared_ptr<http_method_handlers>>    http_api_handlers;
 
+struct HttpService;
+struct HV_EXPORT HttpContext {
+    HttpService*            service;
+    HttpRequestPtr          request;
+    HttpResponsePtr         response;
+    HttpResponseWriterPtr   writer;
+};
+typedef std::shared_ptr<HttpContext>                    HttpContextPtr;
+typedef std::function<int(const HttpContextPtr& ctx)>   http_handler;
+
 struct HV_EXPORT HttpService {
-    // preprocessor -> api service -> file service -> indexof service -> postprocessor
+    // preprocessor -> processor -> postprocessor
     http_sync_handler   preprocessor;
+    // processor: api_handlers -> staticHandler -> errorHandler
+    http_handler        processor;
     http_sync_handler   postprocessor;
+
     // api service (that is http.APIServer)
     std::string         base_url;
     http_api_handlers   api_handlers;
+
     // file service (that is http.FileServer)
-    std::string document_root;
-    std::string home_page;
-    std::string error_page;
+    http_handler    staticHandler;
+    http_handler    largeFileHandler;
+    std::string     document_root;
+    std::string     home_page;
+    std::string     error_page;
     // indexof service (that is http.DirectoryServer)
-    std::string index_of;
+    std::string     index_of;
+
+    http_handler    errorHandler;
 
     HttpService() {
         preprocessor = NULL;
+        processor = NULL;
         postprocessor = NULL;
+
         // base_url = DEFAULT_BASE_URL;
+
+        staticHandler = NULL;
+        largeFileHandler = NULL;
+
         document_root = DEFAULT_DOCUMENT_ROOT;
         home_page = DEFAULT_HOME_PAGE;
         // error_page = DEFAULT_ERROR_PAGE;
+        // index_of = DEFAULT_INDEXOF_DIR;
+
+        errorHandler = NULL;
     }
 
     void AddApi(const char* path, http_method method, http_sync_handler handler = NULL, http_async_handler async_handler = NULL);