手把手写C++服务器(35)：手撕代码——高并发高QPS技术基石之非阻塞send万字长文
Posted 沉迷单车的追风少年
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了手把手写C++服务器(35)：手撕代码——高并发高QPS技术基石之非阻塞send万字长文相关的知识，希望对你有一定的参考价值。
本系列文章导航： 手把手写C++服务器(0)：专栏文章-汇总导航【更新中】
前言：创建socket是默认阻塞的。但是在高并发多QPS的场景中，阻塞模式会极大程度上影响并发性，使之并发名存实亡。而send函数本质上并不是在网络上发送数据，而是将发送缓冲区的数据拷贝到数据内核中；recv函数的本质是将内核缓冲区中的数据拷贝到应用程序的缓冲区中。因此，当缓冲区满了的时候，阻塞/阻塞会影响send/recv的调用，我们这一篇文章重点讨论这一种情况；后面几讲会继续讨论非阻塞connect及与IO复用结合。
目录
预备知识
1、send/recv的本质
send和recv并不是直接收发数据，send函数本质上并不是在网络上发送数据，而是将发送缓冲区的数据拷贝到数据内核中；recv函数的本质是将内核缓冲区中的数据拷贝到应用程序的缓冲区中。
什么时候拷贝、什么时候正式发送，需要看是否启用了nagel算法。如果禁用了nagel算法，存放到内核缓冲区的数据会被立即发出去；有兴趣的同学可以去看一下nagel算法，这也是面试中经常考察的知识点。
如果数据缓冲区满了，阻塞模式和非阻塞模式的表现是不一样的：
- 当socket是阻塞模式时，程序会阻塞在send/recv处。
- 当socket是非阻塞模式时，send/recv不会阻塞当前程序执行流，会立即返回错误：EWOULDBLOCK或EAGAIN。后面我们会用实验验证这一点。
2、文件描述符控制函数：fcntl()
不太清楚的小伙伴请看往期：手把手写C++服务器(27)：五大文件描述符零拷贝、fcntl控制总结_沉迷单车的追风少年-CSDN博客
file control，文件描述符控制。与之类似的系统调用是ioctl。
#include<unistd.h>
#include<fcntl.h>
int fcntl(int fd, int cmd);
int fcntl(int fd, int cmd, long arg);
int fcntl(int fd, int cmd ,struct flock* lock);参数详解
3、设置文件描述符为非阻塞模式
#include <fcntl.h>
// 将文件描述符设置为非阻塞模式
int setnoblocking(int fd) {
// 获取文件描述符旧的状态标志
int old_option = fcntl(fd, F_GETFL);
// 设置非阻塞标志
int new_option = old_option | O_NONBLOCK;
// 设置非阻塞模式
fcntl(fd, F_SETFL, new_option);
// 返回文件描述符旧的状态标志，以便日后恢复改状态标志
return old_option;
}4、服务端、客户端响应/请求一般框架
不太清楚的小伙伴请看往期：https://xduwq.blog.csdn.net/article/details/119482107
服务端
// 设置非阻塞模式
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <assert.h>
#include <errno.h>
#include <netinet/in.h>
int main(int argc, char* argv[]) {
if (argc <= 1) {
printf("error! please input port!\\n");
return 1;
}
//创建监听socket
int listenfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
assert(serv_sock);
int port = atoi(argv[1]);
//将套接字和IP、端口绑定
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr)); //每个字节都用0填充
serv_addr.sin_family = AF_INET; //使用IPv4地址
serv_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); //具体的IP地址
serv_addr.sin_port = htons(port); //端口
if (bind(listenfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) == -1) {
std::cout << "bind listen socket is error" << std::endl;
close(listenfd);
return -1;
}
//进入监听状态，等待用户发起请求
if (listen(listenfd, 20) == -1) {
std::cout << "listen error" << std::endl;
close(listenfd);
return -1;
}
//接收客户端请求
struct sockaddr_in clnt_addr;
socklen_t clnt_addr_size = sizeof(clnt_addr);
int connfd = 0;
for ( ; ; ) {
if ((connfd = accept(listenfd, (struct sockaddr*)&clnt_addr, &clnt_addr_size)) < 0) {
printf("accept error: %s\\n",strerror(errno));
return 1;
}
}
close(listenfd);
return 0;
}客户端
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <iostream>
using namespace std;
int main(int argc, char* argv[]){
if (argc <= 1) {
printf("error! please input port!\\n");
return 1;
}
//创建socket
int clientfd = socket(AF_INET, SOCK_STREAM, 0);
if (clientfd == -1) {
std::cout << "create client error" << std::endl;
return -1;
}
int port = atoi(argv[1]);
//向服务器（特定的IP和端口）发起请求
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr)); //每个字节都用0填充
serv_addr.sin_family = AF_INET; //使用IPv4地址
serv_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); //具体的IP地址
serv_addr.sin_port = htons(port); //端口
int connectfd = connect(clientfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
if (connectfd == -1) {
std::cout << "connect error" << std::endl;
close(clientfd);
return -1;
}
//读取服务器传回的数据
char buffer[40];
read(clientfd, buffer, sizeof(buffer)-1);
printf("Message form server: %s\\n", buffer);
//关闭套接字
close(clientfd);
return 0;
}正式开始
1、客户端
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <iostream>
#include <fcntl.h>
// #define SEND_DATA "NO BLOCKING MODEL"
#define SEND_DATA "hello"
using namespace std;
// 将文件描述符设置为非阻塞模式
int setnoblocking(int fd) {
// 获取文件描述符旧的状态标志
int old_option = fcntl(fd, F_GETFL);
// 设置非阻塞标志
int new_option = old_option | O_NONBLOCK;
// 设置非阻塞模式
if (fcntl(fd, F_SETFL, new_option) == -1) {
// std::cout << "set no blocking model is error" << std::endl;
return -1;
}
// 返回文件描述符旧的状态标志，以便日后恢复改状态标志
return old_option;
}
int main(int argc, char* argv[]) {
if (argc <= 1) {
printf("error! please input port!\\n");
return 1;
}
//创建socket
int clientfd = socket(AF_INET, SOCK_STREAM, 0);
if (clientfd == -1) {
std::cout << "create client error" << std::endl;
return -1;
}
// 第一个入参是端口
int port = atoi(argv[1]);
//向服务器（特定的IP和端口）发起请求
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr)); //每个字节都用0填充
serv_addr.sin_family = AF_INET; //使用IPv4地址
serv_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); //具体的IP地址
serv_addr.sin_port = htons(port); //端口
int connectfd = connect(clientfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
if (connectfd == -1) {
std::cout << "connect error" << std::endl;
close(clientfd);
return -1;
}
std::cout << "old clientfd option is " << fcntl(clientfd, F_GETFL, 0) << std::endl;
// 连接成功后再将clientfd设置为非阻塞模式
int oldconnectfd = setnoblocking(clientfd);
if (oldconnectfd == -1) {
std::cout << "set no block model is error" << std::endl;
close(clientfd);
return -1;
}
// // 获取文件描述符旧的状态标志
// int old_option = fcntl(clientfd, F_GETFL, 0);
// // 设置新的非阻塞标志
// int new_option = old_option | O_NONBLOCK;
// // 设置非阻塞模式
// if (fcntl(clientfd, F_SETFL, new_option) == -1) {
// std::cout << "set no blocking model is error" << std::endl;
// return -1;
// }
std::cout << "new clientfd option is " << fcntl(clientfd, F_GETFL) << std::endl;
std::cout << "new connectfd option is " << fcntl(connectfd, F_GETFL) << std::endl;
// return 0;
// 不停地向服务器发送数据，直到出错退出
int count = 0;
while (true) {
// std::cout << "count = " << count << std::endl;
int ret = send(clientfd, SEND_DATA, strlen(SEND_DATA), 0);
if (ret == -1) {
// 非阻塞模式下，send由于TCP窗口太小发送不出去数据，错误码是EWOULDBLOCK
if (errno == EWOULDBLOCK) {
std::cout << "send data error because TCP windows is too small!" << std::endl;
continue;
} else if (errno == EINTR) {
std::cout << "sending is interupted by signal" << std::endl;
continue;
} else {
std::cout << "send data error" << std::endl;
break;
}
} else if (ret == 0) {
// 对方关闭了连接
std::cout << "send data error" << std::endl;
close(clientfd);
break;
} else {
count++;
std::cout << "send data successfully, count = " << count << std::endl;
}
}
//关闭套接字
close(clientfd);
return 0;
}2、服务端
// 设置非阻塞模式
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <assert.h>
#include <errno.h>
#include <netinet/in.h>
#include <iostream>
int main(int argc, char* argv[]) {
if (argc <= 1) {
printf("error! please input port!\\n");
return 1;
}
// 创建监听socket
int listenfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
assert(listenfd);
int port = atoi(argv[1]);
// 将套接字和IP、端口绑定
struct sockaddr_in serv_addr;
memset(&serv_addr, 0, sizeof(serv_addr)); //每个字节都用0填充
serv_addr.sin_family = AF_INET; //使用IPv4地址
serv_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); //具体的IP地址
serv_addr.sin_port = htons(port); //端口
if (bind(listenfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) == -1) {
std::cout << "bind listen socket is error" << std::endl;
close(listenfd);
return -1;
}
// 进入监听状态，等待用户发起请求
if (listen(listenfd, SOMAXCONN) == -1) {
std::cout << "listen error" << std::endl;
close(listenfd);
return -1;
}
// 接收客户端请求
while (true) {
struct sockaddr_in clientaddr;
socklen_t clientaddrlen = sizeof(clientaddr);
int clientfd = accept(listenfd, (struct sockaddr *)& clientaddr, &clientaddrlen);
if (clientfd != -1) {
std::cout << "accept a client connection" << std::endl;
} else {
std::cout << "accept connection error!" << std::endl;
}
}
close(listenfd);
return 0;
}
3、实验结果
服务端：
./server 1234
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection
accept a client connection客户端：
send data error because TCP windows is too small!
send data error because TCP windows is too small!
send data error because TCP windows is too small!
send data error because TCP windows is too small!
send data error because TCP windows is too small!tcpdump抓包：
tcpdump -i any -nn -S 'tcp port 1234'
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on any, link-type LINUX_SLL (Linux cooked), capture size 262144 bytes
11:32:11.355877 IP 127.0.0.1.50750 > 127.0.0.1.1234: Flags [.], ack 2208883600, win 512, options [nop,nop,TS val 3367396090 ecr 3367340791], length 0
11:32:11.355888 IP 127.0.0.1.1234 > 127.0.0.1.50750: Flags [.], ack 1270908893, win 0, options [nop,nop,TS val 3367396090 ecr 3367285075], length 0
11:32:13.399881 IP 127.0.0.1.39942 > 127.0.0.1.1234: Flags [.], ack 3715653001, win 512, options [nop,nop,TS val 3367398134 ecr 3367291639], length 0
11:32:13.399895 IP 127.0.0.1.1234 > 127.0.0.1.39942: Flags [.], ack 1174986492, win 0, options [nop,nop,TS val 3367398134 ecr 3367184099], length 0
11:33:04.599897 IP 127.0.0.1.44602 > 127.0.0.1.1234: Flags [.], ack 89987839, win 512, options [nop,nop,TS val 3367449334 ecr 3367338743], length 0
11:33:04.599939 IP 127.0.0.1.1234 > 127.0.0.1.44602: Flags [.], ack 1065986402, win 0, options [nop,nop,TS val 3367449334 ecr 3367227655], length 0
11:33:59.895885 IP 127.0.0.1.50750 > 127.0.0.1.1234: Flags [.], ack 2208883600, win 512, options [nop,nop,TS val 3367504630 ecr 3367396090], length 0
11:33:59.895911 IP 127.0.0.1.1234 > 127.0.0.1.50750: Flags [.], ack 1270908893, win 0, options [nop,nop,TS val 3367504630 ecr 3367285075], length 0
11:34:49.409784 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [S], seq 170000750, win 65495, options [mss 65495,sackOK,TS val 3367554143 ecr 0,nop,wscale 7], length 0
11:34:49.409796 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [S.], seq 2701108373, ack 170000751, win 65483, options [mss 65495,sackOK,TS val 3367554143 ecr 3367554143,nop,wscale 7], length 0
11:34:49.409804 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367554143 ecr 3367554143], length 0
11:34:49.409822 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000751:170000756, ack 2701108374, win 512, options [nop,nop,TS val 3367554143 ecr 3367554143], length 5
11:34:49.409861 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000756:170000761, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554143], length 5
11:34:49.409881 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000766:170000771, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.409896 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000776:170000781, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.409911 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000786:170000791, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.409926 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000796:170000801, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.409935 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170000806, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 0
11:34:49.409948 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000811:170000816, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.409963 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000821:170000826, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.409970 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170000826:170000831, ack 2701108374, win 512, options [nop,nop,TS val 3367554144 ecr 3367554144], length 5
11:34:49.429146 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], seq 170000836:170033604, ack 2701108374, win 512, options [nop,nop,TS val 3367554163 ecr 3367554144], length 32768
11:34:49.447874 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], seq 170000836:170033604, ack 2701108374, win 512, options [nop,nop,TS val 3367554182 ecr 3367554144], length 32768
11:34:49.447940 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170033604, win 375, options [nop,nop,TS val 3367554182 ecr 3367554144,nop,nop,sack 1 {170000836:170033604}], length 0
11:34:49.447953 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], seq 170033604:170066372, ack 2701108374, win 512, options [nop,nop,TS val 3367554182 ecr 3367554182], length 32768
11:34:49.491875 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170066372, win 119, options [nop,nop,TS val 3367554226 ecr 3367554182], length 0
11:34:49.707879 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [P.], seq 170066372:170081604, ack 2701108374, win 512, options [nop,nop,TS val 3367554442 ecr 3367554226], length 15232
11:34:49.707893 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170081604, win 0, options [nop,nop,TS val 3367554442 ecr 3367554442], length 0
11:34:49.923893 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367554658 ecr 3367554442], length 0
11:34:49.923923 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170081604, win 0, options [nop,nop,TS val 3367554658 ecr 3367554442], length 0
11:34:50.359883 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367555094 ecr 3367554658], length 0
11:34:51.223886 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367555958 ecr 3367554658], length 0
11:34:51.223897 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170081604, win 0, options [nop,nop,TS val 3367555958 ecr 3367554442], length 0
11:34:52.955881 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367557690 ecr 3367555958], length 0
11:34:52.955893 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170081604, win 0, options [nop,nop,TS val 3367557690 ecr 3367554442], length 0
11:34:53.310633 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [S], seq 4126399068, win 65495, options [mss 65495,sackOK,TS val 3367558044 ecr 0,nop,wscale 7], length 0
11:34:53.310651 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [S.], seq 2899400818, ack 4126399069, win 65483, options [mss 65495,sackOK,TS val 3367558044 ecr 3367558044,nop,wscale 7], length 0
11:34:53.310662 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 0
11:34:53.310737 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399069:4126399074, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310742 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126399074, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 0
11:34:53.310786 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399074:4126399079, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310790 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126399079, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 0
11:34:53.310797 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399079:4126399084, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310799 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126399084, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 0
11:34:53.310805 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399084:4126399089, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310812 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399089:4126399094, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310820 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399094:4126399099, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310828 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399099:4126399104, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310844 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126399114, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 0
11:34:53.310858 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399119:4126399124, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310865 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399124:4126399129, ack 2899400819, win 512, options [nop,nop,TS val 3367558044 ecr 3367558044], length 5
11:34:53.310880 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399134:4126399139, ack 2899400819, win 512, options [nop,nop,TS val 3367558045 ecr 3367558045], length 5
11:34:53.310888 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399139:4126399144, ack 2899400819, win 512, options [nop,nop,TS val 3367558045 ecr 3367558045], length 5
11:34:53.310895 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399144:4126399149, ack 2899400819, win 512, options [nop,nop,TS val 3367558045 ecr 3367558045], length 5
11:34:53.310902 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126399149:4126399154, ack 2899400819, win 512, options [nop,nop,TS val 3367558045 ecr 3367558045], length 5
11:34:53.317094 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], seq 4126399154:4126431922, ack 2899400819, win 512, options [nop,nop,TS val 3367558051 ecr 3367558045], length 32768
11:34:53.331887 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], seq 4126399154:4126431922, ack 2899400819, win 512, options [nop,nop,TS val 3367558066 ecr 3367558045], length 32768
11:34:53.331902 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126431922, win 375, options [nop,nop,TS val 3367558066 ecr 3367558045,nop,nop,sack 1 {4126399154:4126431922}], length 0
11:34:53.331918 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], seq 4126431922:4126464690, ack 2899400819, win 512, options [nop,nop,TS val 3367558066 ecr 3367558066], length 32768
11:34:53.379882 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126464690, win 119, options [nop,nop,TS val 3367558113 ecr 3367558066], length 0
11:34:53.595879 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [P.], seq 4126464690:4126479922, ack 2899400819, win 512, options [nop,nop,TS val 3367558329 ecr 3367558113], length 15232
11:34:53.595894 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126479922, win 0, options [nop,nop,TS val 3367558330 ecr 3367558329], length 0
11:34:53.807899 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367558542 ecr 3367558330], length 0
11:34:53.807936 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126479922, win 0, options [nop,nop,TS val 3367558542 ecr 3367558329], length 0
11:34:54.235890 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367558969 ecr 3367558542], length 0
11:34:55.095891 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367559829 ecr 3367558542], length 0
11:34:55.095915 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126479922, win 0, options [nop,nop,TS val 3367559830 ecr 3367558329], length 0
11:34:56.471880 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367561205 ecr 3367557690], length 0
11:34:56.471900 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170081604, win 0, options [nop,nop,TS val 3367561206 ecr 3367554442], length 0
11:34:56.791884 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367561525 ecr 3367559830], length 0
11:34:56.791905 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126479922, win 0, options [nop,nop,TS val 3367561526 ecr 3367558329], length 0
11:35:00.311890 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367565045 ecr 3367561526], length 0
11:35:00.311917 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126479922, win 0, options [nop,nop,TS val 3367565046 ecr 3367558329], length 0
11:35:03.383878 IP 127.0.0.1.51346 > 127.0.0.1.1234: Flags [.], ack 2701108374, win 512, options [nop,nop,TS val 3367568117 ecr 3367561206], length 0
11:35:03.383893 IP 127.0.0.1.1234 > 127.0.0.1.51346: Flags [.], ack 170081604, win 0, options [nop,nop,TS val 3367568117 ecr 3367554442], length 0
11:35:07.223885 IP 127.0.0.1.51768 > 127.0.0.1.1234: Flags [.], ack 2899400819, win 512, options [nop,nop,TS val 3367571957 ecr 3367565046], length 0
11:35:07.223906 IP 127.0.0.1.1234 > 127.0.0.1.51768: Flags [.], ack 4126479922, win 0, options [nop,nop,TS val 3367571957 ecr 3367558329], length 0
11:35:09.105206 IP 127.0.0.1.53466 > 127.0.0.1.1234: Flags [S], seq 1554713212, win 65495, options [mss 65495,sackOK,TS val 3367573839 ecr 0,nop,wscale 7], length 0
11:35:09.105220 IP 127.0.0.1.1234 > 127.0.0.1.53466: Flags [S.], seq 2146429495, ack 1554713213, win 65483, options [mss 65495,sackOK,TS val 3367573839 ecr 3367573839,nop,wscale 7], length 0参考：
以上是关于手把手写C++服务器(35)：手撕代码——高并发高QPS技术基石之非阻塞send万字长文的主要内容，如果未能解决你的问题，请参考以下文章
手把手写C++服务器(37)：手撕代码——高并发多线程技术基石之异步connect万字长文
手把手写C++服务器(34)：高并发高吞吐IO秘密武器——epoll池化技术两万字长文
手把手写C++服务器(30)：手撕代码——基于TCP/IP的抛弃服务discard