自主WebServer实现

Posted 2022-12-07 捕获一只小肚皮

项目介绍

该项目是一个基于http和tcp协议自主实现的WebServer,用于实现浏览器对服务器的简单请求,然后接收服务器返回的处理请求数据,目的是为了让学习者更好的对网络连接和相关协议进行理解,同时制作出像网络在线计算机,音视频请求播放,或者类似博客的功能,该醒目利用到的主要技术有网络编程(socket),多线程编程,cgi技术,管道通信等

项目框架

TCP_SERVER的搭建

主要是进行监听套接字的创建,然后绑定IP,进行监听,由于是在多线程环境下,我们对tcp_server使用了单例模式

#pragma  once
#include <iostream>
#include "LOG.hpp"
#include <sys/types.h>
#include <sys/socket.h>
#include <cstdlib>
#include <cstring>
#include <pthread.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#define PORT 8081
#define BACKLOG 5
class TcpServer
    private:
        int listen_sock;
        int port;
        static TcpServer* tcpserver;
    private:
        TcpServer(int _port = PORT):listen_sock(),port(_port) 
        ~TcpServer()
    public:
        static TcpServer* GetInstance(int _port = PORT)
            static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
            if(tcpserver == nullptr)
                pthread_mutex_lock(&lock);
                if(tcpserver == nullptr)
                    tcpserver = new TcpServer(_port);
                    tcpserver->InitTcpServer();
                
                pthread_mutex_unlock(&lock);
            
            return tcpserver;
        
        void InitTcpServer()
            CreateSock();
            BindSock();
            ListenSock();
        
        void CreateSock()
            listen_sock = socket(AF_INET,SOCK_STREAM,0);
            if(listen_sock < 0)
                LOG(INFO,"Create Sock Unsucess!");
                exit(1);
            
            int opt = 1;
            setsockopt(listen_sock,SOL_SOCKET,SO_REUSEADDR,&opt,sizeof(opt));
            LOG(INFO,"Create Sock Sucess!");
        
        void BindSock()
            struct sockaddr_in local;
            memset(&local,0,sizeof(local));
            local.sin_family = AF_INET;
            local.sin_port = htons(port);
            local.sin_addr.s_addr = INADDR_ANY;  //云服务器不可以直接绑定ip
            if(bind(listen_sock,(struct sockaddr*)&local,sizeof(local))<0)
                LOG(INFO,"Bind Sock Unsucess!");
                exit(2);
            
            LOG(INFO,"Bind Sock Sucess!");
        

        void ListenSock()
            if(listen(listen_sock,BACKLOG)<0)
                LOG(INFO,"Listen Sock Unsucess!");
                exit(3);
            
            LOG(INFO,"Listen Sock Sucess!");
        

        int GetListenSock()
            return listen_sock;
        
        
;
TcpServer*  TcpServer::tcpserver = nullptr;

http_server的搭建

http_server主要是调用tcp_server进行获取监听套接字,然后接收请求并利用线程池技术进行执行请求

#pragma once 
#include "TcpServer.hpp"
#include "Protocol.hpp"
#include "LOG.hpp"
#include "Task.hpp"
#include "ThreadPool.hpp"

#define PORT 8081
class HttpServer
    private:
        int sock;
        int port;
        int stop;
        TcpServer* tcpserver;
    public:
        HttpServer(int _port = PORT):port(_port),stop(false),tcpserver(nullptr)
        void InitHttpServer()
            // 防止对端(client and server)断开发送SIGPIPE信号时,执行默认动作
            signal(SIGPIPE,SIG_IGN);
        

        void Loop()
            tcpserver = TcpServer::GetInstance(port);
            while(!stop)
                struct sockaddr_in peer;
                socklen_t len = sizeof(peer);
                memset(&peer,0,sizeof(peer));
                sock = accept(tcpserver->GetListenSock(),(struct sockaddr*)&peer,&len);
                LOG(INFO,"Begin Get A New Link!");
                if(sock < 0)
                    LOG(INFO,"Accept Socket Unsucess!");
                    continue;
                
                LOG(INFO,"Begin Proess Request!");
                TASK task(sock);
                ThreadPool::GetInstance()->PushTask(task);
            
        
       ~HttpServer()
;

日志系统

主要用来对错误时间进行记录,当程序执行错误时候,可以检查日志系统进行查看排错,然后修正

#pragma  once
#include <string>
#include <iostream>
#include <cstdio>
#include <time.h>
#define INFO    1
#define WARNING 2
#define ERROR   3
#define FATAL   4
#define LOG(level,message) log(#level,message,__FILE__,__LINE__)
void log(std::string level,std::string message,std::string fname,int eline)
        struct tm t;
        time_t now = time(NULL);
        localtime_r(&now,&t);

        printf("[%-7s][%-4d-%02d-%02d %02d:%02d:%02d][%-30s][%-15s][%-4d]\\n",\\
                level.c_str(),t.tm_year+1900,t.tm_mon+1,t.tm_mday,t.tm_hour,t.tm_min,t.tm_sec,\\
                message.c_str(),fname.c_str(),eline);
    

Protocol的搭建

该项目主要分为三个模块,分别是对http的请求进行分析:检测出请求方法,uri等资源路径,请求版本,请求正文;然后按照响应模板进行构建响应;然后返还给客户端;

class EndPoint
    private:
        int sock;
        bool stop;
        HttpRequest http_request;
        HttpResbonse http_resbonse;
    public:
        EndPoint(int _sock):sock(_sock),stop(false)


        int IsStop()
            return stop;
         

        void RecvHttpRequest()
            if((!Recv_Parse_HttpRequest_Line()) && \\
                (!Recv_Parse_HttpRequest_Head()) && \\
                (!Recv_HttpRequest_Body()))
            

        int  Recv_Parse_HttpRequest_Line()
            LOG(INFO,"Begin Recv Request"); 
            //recv
            if(Utility::ReadLine(sock,http_request.request_line)> 0)
                http_request.request_line.back() = 0;
                LOG(INFO,http_request.request_line);
            
            else
                stop = true;
                return stop;
            
            //parse
            std::stringstream ss(http_request.request_line);
            ss>>http_request.method>>http_request.uri>>http_request.version;
            //toupper
            auto& mt = http_request.method;
            std::transform(mt.begin(),mt.end(),mt.begin(),::toupper);

            size_t pos = http_request.path.rfind(".");
            if(pos != std::string::npos)
                http_request.suffix = http_request.path.substr(pos+1);
            
            return stop;
        

        int Recv_Parse_HttpRequest_Head()
            //recv
            std::string line = "";
            while(true)
                line.clear();
                if(Utility::ReadLine(sock,line)<=0)
                    stop = true;
                    return stop;
                
                if(line == "\\n")
                    http_request.blank = line;
                    break;
                
                line.back() = 0;
                http_request.request_header.push_back(line);
                LOG(INFO,line);
            

            //parse
            for(auto& item:http_request.request_header) 
                std::string key,value;
                Utility::CutString(item,key,value,": ");
                http_request.request_header_kv[key] = value;
            
            return stop;
        

        int Recv_HttpRequest_Body()
            auto& mt = http_request.method;
            if(mt == "POST")
                auto item = http_request.request_header_kv.find("Content-Length");
                if(item != http_request.request_header_kv.end())
                    http_request.content_length = atoi(item->second.c_str());
                    int size = http_request.content_length;
                    while(size)
                        char ch;
                        size_t s = recv(sock,&ch,1,0);
                        if(s > 0)
                            http_request.request_body.push_back(ch);
                            size--;
                        
                        else if(s == 0)
                            break;
                        
                        else 
                            stop = true;
                            return stop;
                            LOG(ERROR,"Recv RequestBody Error");
                            exit(1);
                        
                    
                
            
            return stop;
        

        void Build_HttpResbonse()
            http_request.path = "wwwroot";
            auto& mt = http_request.method;
            std::string tmp;
            struct stat buf;

            //检查方法是否合法
            if(mt!= "GET" && mt != "POST")
                LOG(INFO,"Request Method Is Not Law!");
                http_resbonse.status_code = NOT_FOUND;
                goto END;
            
            if(mt == "GET")
                //如果是get方法,检查是否带参
                size_t pos = http_request.uri.find("?"); 
                if(pos != std::string::npos)
                    Utility::CutString(http_request.uri,tmp,http_request.query,"?");
                    http_request.cgi = true;
                
                else 
                    tmp = http_request.uri;
                
            
            else if(mt == "POST")
                tmp = http_request.uri;
                http_request.cgi = true;
            
            else


            //拼接web根目录
            http_request.path += tmp;
            if(http_request.path.back() == '/')
                http_request.path += "index.html";
            

            //检查资源是否存在
            if(!stat(http_request.path.c_str(),&buf))
                //检查资源是否是目录
                if(S_ISDIR(buf.st_mode))
                    http_request.path += "/";
                    http_request.path += "index.html";
                    stat(http_request.path.c_str(),&buf);
                //检查资源是否是可自行文件 
                else if((S_IXUSR & buf.st_mode)||(S_IXGRP & buf.st_mode)||(S_IXOTH & buf.st_mode))
                    http_request.cgi = true;
                
                http_request.size = buf.st_size;
            
            else 
                LOG(INFO,http_request.path +"Resorce Is Not Exist");
                http_resbonse.status_code = NOT_FOUND;
                goto END;
            

            if(http_request.cgi)
                http_resbonse.status_code = HandlerCgi();
            
            else
                http_resbonse.status_code = HandlerNonCgi();
            
END:
            Build_HttpResbonseHelper();
        

        int HandlerNonCgi()
            auto& path = http_request.path;
            http_resbonse.fd = open(path.c_str(),O_RDONLY);
            if(http_resbonse.fd > 0)
                return OK;
            
            return NOT_FOUND;

        
        int HandlerCgi()
            auto& bin = http_request.path;
            auto& query_get = http_request.query;
            auto& query_post = http_request.request_body;
            int upfd[2],downfd[2];
            if(pipe(upfd)<0 || pipe(downfd)<0)
                LOG(ERROR,"PIPE ERROR");
                return ERROR_SERVER;
            

            pid_t pid = fork();
            if(pid == 0)
                close(upfd[0]);
                close(downfd[1]);

                dup2(downfd[0],0);
                dup2(upfd[1],1);

                 std::string method = "METHOD=";
                method += http_request.method;
                putenv((char*)method.c_str());          

                if(http_request.method == "GET")
                    std::string query = "QUERY=";
                    query += query_get;
                    putenv((char*)query.c_str());
                
                else if(http_request.method == "POST")
                    int cl = http_request.content_length;
                    std::string content_length = "CONTENT_LENGTH=";
                    content_length += std::to_string(cl);
                    putenv((char*)content_length.c_str());
                

                execl(bin.c_str(),bin.c_str(),nullptr);

                exit(1);
            
            else if(pid > 0)
                close(upfd[1]);
                close(downfd[0]);

                std::string& method = http_request.method;

                if(method == "POST")
                    const char* query = query_post.c_str();
                    ssize_t total = 0,size = 0;
                    while(total < http_request.content_length && \\
                            (size = write(downfd[1],query+total,http_request.content_length - total)) > 0)
                        total += size;
                    
                

                std::string& body = http_resbonse.resbonse_body;
                char ch = 0;
                while(read(upfd[0],&ch,1) > 0)
                    body.push_back(ch);
                
                int status = 0;
                int ret = waitpid(pid,&status,0);
                if(ret == pid)
                    if(WIFEXITED(status) && !WEXITSTATUS(status))
                        return OK;
                     
                    else 
                        LOG(ERROR,"----------");
                        return ERROR_SERVER;
                    
                
                close(upfd[0]);
                close(downfd[1]);
            
            else
                LOG(ERROR,"Create SonProcess Unsucess");
                return 404;
            
        

        void Build_HttpResbonseHelper()
            int code = http_resbonse.status_code;
            auto& status_line = http_resbonse.resbonse_line;
            status_line = VERSION; 
            status_line += " ";
            status_line += std::to_string(code);
            status_line += " ";
            status_line += StatusCodeDesc(code);
            status_line += LINE_END;

            if(code == 200)
                BuildOKResbonse();
            
            else 
                BuildErrorResbonse(code);
            
        
        void BuildOKResbonse()
            std::string line = "Content-Length: ";
            if(http_request.cgi)
                line += std::to_string(http_resbonse.resbonse_body.size());
            
            else 
                line += std::to_string(http_request.size);
            
            line+=LINE_END;
            http_resbonse.resbonse_header.push_back(line);

            line = "Content-Type: ";
            line += SuffixDesc(http_request.suffix);
            line += LINE_END;
            http_resbonse.resbonse_header.push_back(line);
        
        void BuildErrorResbonse(int code)
            http_request.cgi = false;
            std::string page = GetErrorFile(code); 
            std::cout<<"page"<<page<<std::endl;
            http_resbonse.fd = open(page.c_str(),O_RDONLY);
            if(http_resbonse.fd > 0)
                struct stat buf;
                stat(page.c_str(),&buf);
                http_request.size = buf.st_size;

                std::string line = "Content-type: text/html";
                line += LINE_END;
                http_resbonse.resbonse_header.push_back(line);

                line = "Content-Length: ";
                line += std::to_string(buf.st_size);
                line += LINE_END;
                http_resbonse.resbonse_header.push_back(line);
            
            
        

        void Send_HttpResbonse()
            auto& line = http_resbonse.resbonse_line;

            send(sock,line.c_str(),line.size(),0);
            for(auto& item : http_resbonse.resbonse_header)
                send(sock,item.c_str(),item.size(),0);
            
            send(sock,http_resbonse.blank.c_str(),http_resbonse.blank.size(),0);
            if(http_request.cgi)
                send(sock,http_resbonse.resbonse_body.c_str(),http_resbonse.resbonse_body.size(),0);
            
            else 
                sendfile(sock,http_resbonse.fd,nullptr,http_request.size);
                close(http_resbonse.fd);
            
        

        ~EndPoint()
            close(sock);
        
;

CGI技术

CGI（Common Gateway Interface）公共网关接口，是外部扩展应用程序与 Web 服务器交互的一个标准接口。它可以使外部程序处理www上客户端送来的表单数据并对此作出反应，通过某些特定的方式处理数据返回给Web服务器进而返回给client端;

一般是把执行资源交给子进程进行程序替换,而不能直接用程序替换,因为这是多线程环境,只要一个进程,如果使用程序替换,那么当前的整个进程将会崩溃(资源没有了)

而cgi程序需要接收父进程交接过来的数据,这里便采用管道通信方式进行数据传递,但这里有个小问题需要注意,我们必须对文件描述符进行重定向,因为被替换后的程序管道只有0,1,2,原先打开的匿名管道被隐藏起来了.

   int HandlerCgi()
            auto& bin = http_request.path;
            auto& query_get = http_request.query;
            auto& query_post = http_request.request_body;
            int upfd[2],downfd[2];
            if(pipe(upfd)<0 || pipe(downfd)<0)
                LOG(ERROR,"PIPE ERROR");
                return ERROR_SERVER;
            

            pid_t pid = fork();
            if(pid == 0)        //子进程导入环境变量
                close(upfd[0]);
                close(downfd[1]);

                dup2(downfd[0],0);
                dup2(upfd[1],1);

                 std::string method = "METHOD=";
                method += http_request.method;
                putenv((char*)method.c_str());          

                if(http_request.method == "GET")
                    std::string query = "QUERY=";
                    query += query_get;
                    putenv((char*)query.c_str());
                
                else if(http_request.method == "POST")
                    int cl = http_request.content_length;
                    std::string content_length = "CONTENT_LENGTH=";
                    content_length += std::to_string(cl);
                    putenv((char*)content_length.c_str());
                

                execl(bin.c_str(),bin.c_str(),nullptr);

                exit(1);
            
            else if(pid > 0)    //父进程写入数据 和 读取数据
                close(upfd[1]);
                close(downfd[0]);

                std::string& method = http_request.method;

                if(method == "POST")
                    const char* query = query_post.c_str();
                    ssize_t total = 0,size = 0;
                    while(total < http_request.content_length && \\
                            (size = write(downfd[1],query+total,http_request.content_length - total)) > 0)
                        total += size;
                    
                

                std::string& body = http_resbonse.resbonse_body;
                char ch = 0;
                while(read(upfd[0],&ch,1) > 0)
                    body.push_back(ch);
                
                int status = 0;
                int ret = waitpid(pid,&status,0);
                if(ret == pid)
                    if(WIFEXITED(status) && !WEXITSTATUS(status))
                        return OK;
                     
                    else 
                        LOG(ERROR,"----------");
                        return ERROR_SERVER;
                    
                
                close(upfd[0]);
                close(downfd[1]);
            
            else
                LOG(ERROR,"Create SonProcess Unsucess");
                return 404;
            
        

testCGI

#include <iostream>
#include <cstdlib>
#include <string>
#include <unistd.h>
bool GetQuery(std::string& query)
    std::string method = getenv("METHOD");
    if(method == "GET")
        query = getenv("QUERY");
    
    else if(method == "POST")
        int content_length = atoi(getenv("CONTENT_LENGTH"));
        char ch = 0;
        while(content_length)
            ssize_t s = read(0,&ch,1);
            if(s>0)
                content_length--;
                query.push_back(ch);
            
            else if(s == 0)
                break;
            
            else
                return false;
            
        
    
    return true;


void CutQuery(std::string& tar,std::string& key,std::string& value,const std::string& sep)
    size_t pos = tar.find(sep);
    if(pos != std::string::npos)
        key = tar.substr(0,pos);
        value = tar.substr(pos+sep.size());
    


int main()
    std::string query;
    GetQuery(query);

    std::string str1,str2;
    std::string name1,name2;
    std::string value1,value2;
    CutQuery(query,str1,str2,"&");
    CutQuery(str1,name1,value1,"=");
    CutQuery(str2,name2,value2,"=");

    std::cout<<name1<<":"<<value1<<std::endl;
    std::cout<<name2<<":"<<value2<<std::endl;
    return 0;

线程池技术

创建批次num数量的线程进行执行任务队列中的任务,执行逻辑为当任务队列中又任务,那么便竞争锁执行,倘若没有任务,那么所有线程进行等待;

#pragma once
#include <iostream>
#include "Task.hpp"
#include <pthread.h>
#include <queue>
#define NUM 10

class ThreadPool
    private:
        int num;
        std::queue<TASK> TaskQueue; 
        pthread_mutex_t lock;
        pthread_cond_t cond;
        ThreadPool(int _num = NUM):num(_num)
            pthread_mutex_init(&lock,nullptr);
            pthread_cond_init(&cond,nullptr);
        
        ThreadPool(const ThreadPool& TP)
        ~ThreadPool()
            pthread_mutex_destroy(&lock);
            pthread_cond_destroy(&cond);
        
        static ThreadPool* tp;
    public:

        static ThreadPool* GetInstance(int _num = NUM)
           static pthread_mutex_t t = PTHREAD_MUTEX_INITIALIZER;
            if(tp == nullptr)
                pthread_mutex_lock(&t);
                if(tp == nullptr)
                    tp = new ThreadPool(_num);
                    tp->InitThreadPool();
                
                pthread_mutex_unlock(&t);
            
            return tp;
        

        static void* ThreadRoutine(void* arg)
            ThreadPool* tp = (ThreadPool*)arg;
            while(true)
                TASK task;
                tp->Lock();
                while(tp->IsEmpty())
                    tp->ThreadWait();
                
                tp->PopTask(task);
                tp->Unlock();
                task.ProcessOn();
            
        
        bool InitThreadPool()
            for(int i = 0; i< num;i++)
                pthread_t tid;
                if(0 != pthread_create(&tid,nullptr, ThreadRoutine,this))
                    LOG(FATAL,"create pthread unsucess");
                    return false;
                
            
            return true;
        
        void Lock()
            pthread_mutex_lock(&lock);
        
        void Unlock()
            pthread_mutex_unlock(&lock);
        
        void ThreadWait()
            pthread_cond_wait(&cond,&lock);
        
        void ThreadWakeup()
            pthread_cond_signal(&cond);
        
        bool IsEmpty()
            return TaskQueue.size() == 0 ? true:false;
        
        void PushTask(const TASK& task)
            Lock();
            TaskQueue.push(task);
            Unlock();                             
            ThreadWakeup();
            //Routine函数中cond_wait用while循环判断主要是解锁和唤醒代码的顺序问题
            //如果先解锁,那么就会有其他进程抢到锁开始执行后续,但不一定是等待进程在执行
            //所以当后面唤醒等待进程时候,可能会发现任务队列仍然为空,但却继续往下执行,这不符合我们的逻辑
        
        void PopTask(TASK& task)
            task = TaskQueue.front();
            TaskQueue.pop();
        
;
ThreadPool* ThreadPool::tp = nullptr;

项目总结

聚焦于处理HTTP的请求和构建对应响应; 我们主要研究基于 HTTP/1.0 短连接 的GET和POST方法;

获得请求，分析请求，错误处理等; 制定特定的网页src用于返回; 引入简单的日志系统

搭建CGI机制;

父子管道，设计dup2重定向，环境变量传参等

引入线程池;

采用多线程技术，缓解内存开销.

源码链接:源码