F-Stack实现UDP服务端客户端，并进行吞吐量测试的实现
Posted rtoax
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了F-Stack实现UDP服务端客户端，并进行吞吐量测试的实现相关的知识，希望对你有一定的参考价值。
Table of Contents
最近在做DPDK相关的网口加速的内容，开始采用DPDK+VPP的方式，但鉴于VPP的部署需要花费一定的时间，遂采用F-Stack，关于F-Stack，本文不做介绍，只说明它是腾讯开发的。
鉴于“一图胜过千言”，本文将以图+简短的话为例
怎么个加速法？
F-Stack不走内核协议栈，通过移植BSD的协议栈和DPDK进行联合，组成一套完整的用户态协议栈。
我准备用两个万兆网卡加速，如下图
然后在采用一个网卡运行服务端，一个网卡运行客户端，
也可以总结成这样
可是最后都初始化ff_init后法宝不成功，最终我确定了一个方案
也就是万兆网卡运行客户端进行收发包，内核协议栈控制的千兆网卡运行客户端收发包。
我的网络拓扑
一开始我的拓扑是下面这样
我从我的虚拟机进行对左侧X86的控制，左面服务器运行客户端和服务端，但是好像哪里不对，因为我要让数据像下面这样走
当然是少了一根线，可以有两种连接方法
我踩用第一种，如下
开始编码
https://download.csdn.net/download/Rong_Toa/12631153
服务端
server.c
/**
* 测试F-Stack的UDP发包速率
* 作者：荣涛 <rongtao@sylincom.com>
* 时间：2020年7月16日10:08:34
*/
#include "common.h"
#include <strings.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <assert.h>
#include <sys/socket.h>
#include <netinet/ip.h>
#include <sys/epoll.h>
int sockfd;
int epfd;
struct epoll_event ev;
struct epoll_event events[MAX_EVENTS];
char buf[MAXLINE] = "100000000";
int udpsocket_server()
int sockfd;
struct sockaddr_in servaddr;
bzero(&servaddr, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
servaddr.sin_port = htons(PORT);
if((sockfd = ff_socket(AF_INET, SOCK_DGRAM, 0)) < 0)
return -1;
if(ff_bind(sockfd, (struct linux_sockaddr *)&servaddr, sizeof(servaddr)))
return -1;
return sockfd;
long int direct_sendto_addr(int sockfd, const char *dst_ip, int port, long int n_pkg)
struct sockaddr_in cliaddr;
int n = 0;
long int npkg = 0, nbyte=0;
socklen_t len;
len = sizeof(cliaddr);
struct timeval tvbrfore, tvafter;
bzero(&cliaddr, sizeof(struct sockaddr_in));
cliaddr.sin_family = AF_INET;
cliaddr.sin_port = htons(port);
inet_pton(AF_INET, dst_ip, &cliaddr.sin_addr);
/* 统计时间 */
gettimeval(&tvbrfore);
/* 循环发包，测速率 */
while(1)
if((n = ff_sendto(sockfd, buf, MAXLINE, 0, (struct linux_sockaddr *)&cliaddr, len)) < 0)
log_warn("sendto error");
exit(1);
else
nbyte += n;
if(npkg++ > n_pkg)
break;
/* 统计时间 */
gettimeval(&tvafter);
/* 输出此段时间内的速率 */
statistic_throughput("Sendto", &tvbrfore, &tvafter, nbyte, npkg-1);
int server_loop(void *arg)
int n;
socklen_t len;
struct sockaddr_in cliaddr;
struct timeval tvbrfore, tvafter;
#if HAVE_FSTACK!=1
while(1)
log_warn("Wait for a Client epoll_wait...\\n");
#endif
/* Wait for events to happen */
int nevents = ff_epoll_wait(epfd, events, MAX_EVENTS, -1);
int i;
len = sizeof(cliaddr);
#if HAVE_FSTACK!=1
log_warn("nevents = %d\\n", nevents);
#endif
for (i = 0; i < nevents; ++i)
/* 如果是UDP的服务端， 接收一个来自客户端的消息，
获取客户端地址，用于向客户端发送数据测发送速率 */
if (events[i].data.fd == sockfd)
if((n = ff_recvfrom(sockfd, buf, MAXLINE, 0, (struct linux_sockaddr *)&cliaddr, &len)) < 0)
log_warn("recvfrom error\\n");
else
log_warn("Server recv %s\\n", buf);
log_warn("Client Family %d\\n", cliaddr.sin_family);
log_warn("Client Port %u\\n", ntohs(cliaddr.sin_port));
log_warn("Client Addr %s\\n", inet_ntoa(cliaddr.sin_addr));
long int npkg = 0, nbyte=0;
long int Nkpg = atol(buf);
#if HAVE_FSTACK==1
Nkpg = Nkpg>1000000?Nkpg:1000000;
#else
Nkpg = Nkpg>10000?Nkpg:10000;
#endif
log_warn("Ready Send N Pkg %ld\\n", Nkpg);
/* 告诉我向客户端发了啥 */
#define MY_ID "[FSTACK-DPDK][RongTao Test][牛逼了老哥]"
strcpy(buf, MY_ID);
/* 统计时间 */
gettimeval(&tvbrfore);
/* 循环发包，测速率 */
while(1)
if((n = ff_sendto(sockfd, buf, MAXLINE, 0, (struct linux_sockaddr *)&cliaddr, len)) < 0)
log_warn("sendto error");
exit(1);
else
nbyte += n;
if(npkg++ > Nkpg)
break;
/* 统计时间 */
gettimeval(&tvafter);
/* 输出此段时间内的速率 */
statistic_throughput("Sendto", &tvbrfore, &tvafter, nbyte, npkg-1);
#if HAVE_FSTACK!=1
#endif
int main(int argc, char *argv[])
/* 绑核 */
setaffinity(7);
#if HAVE_FSTACK==1
ff_init(argc, argv);
#endif
log_warn("Init socket.\\n");
sockfd = udpsocket_server();
assert((epfd = ff_epoll_create(10)) > 0);
ev.data.fd = sockfd;
ev.events = EPOLLIN;
ff_epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev);
log_warn("Sockfd %d Wait for a Client...\\n", sockfd);
#if 1
#if HAVE_FSTACK==1
log_warn("Ready to ff_run.\\n");
ff_run(server_loop, NULL);
#else
server_loop(NULL);
#endif
#else //直接发送
direct_sendto_addr(sockfd, "10.170.6.66", PORT, 10000);
#endif
common.h
/**
* 测试F-Stack的UDP发包速率
* 作者：荣涛 <rongtao@sylincom.com>
* 时间：2020年7月16日10:08:34
*/
#ifndef __COMMON_H
#define __COMMON_H 1
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/sysinfo.h>
#include <time.h>
#include <stdint.h>
#define __USE_GNU
#include <sched.h>
#include <ctype.h>
#include <string.h>
#include <stdarg.h>
#include <libgen.h>
#include "ff_config.h"
#include "ff_api.h"
#include "ff_epoll.h"
/* 使用 fstack 进行UDP发包速率测试=1
使用 linux 进行UDP发包速率测试=1
*/
#define HAVE_FSTACK 1
#define PORT 2152
#define MAXLINE 1500
#define MAX_EVENTS 10
#define TEST_ADDR1 "10.170.7.169"
#define TEST_ADDR2 "10.170.7.170"
/**
* 打印log debug信息
* 作者： 荣涛
* 时间： s2020年7月15日10:12:24
*/
enum
__LV_INFO,
__LV_WARNING,
__LV_ERR,
__LV_DEBUG,
;
#define LOG_DEBUG 1
#ifdef LOG_DEBUG
#define log_info(fmt...) ___debug_log(__LV_INFO, __FILE__, __func__ ,__LINE__, fmt)
#define log_warn(fmt...) ___debug_log(__LV_WARNING, __FILE__, __func__ ,__LINE__, fmt)
#define log_error(fmt...) ___debug_log(__LV_ERR, __FILE__, __func__ ,__LINE__, fmt)
#define log_debg(fmt...) ___debug_log(__LV_DEBUG, __FILE__, __func__ ,__LINE__, fmt)
#else
#define log_info(fmt...)
#define log_warn(fmt...)
#define log_error(fmt...)
#define log_debg(fmt...)
#define log_errorno(i_errno)
#endif
#if HAVE_FSTACK!=1
#define ff_socket socket
#define ff_bind bind
#define ff_recvfrom recvfrom
#define ff_sendto sendto
#define linux_sockaddr sockaddr
#define ff_epoll_wait epoll_wait
#define ff_epoll_create epoll_create
#define ff_epoll_ctl epoll_ctl
#endif
static inline int ___debug_log(int level, char *file, const char *func, int line, char *fmt, ...)
va_list av;
va_start(av, fmt);
switch(level)
case __LV_INFO:
printf(" [%sINFO%s][%s:%s:%d]: ","\\033[1;36m","\\033[0m", basename(file), func, line);
break;
case __LV_WARNING:
printf(" [%sWARN%s][%s:%s:%d]: ","\\033[1;35m","\\033[0m", basename(file), func, line);
break;
case __LV_ERR:
printf("[%sERROR%s][%s:%s:%d]: ","\\033[1;31m","\\033[0m", basename(file), func, line);
break;
case __LV_DEBUG:
printf("[%sDEBUG%s][%s:%s:%d]: ","\\033[1m", "\\033[0m", basename(file), func, line);
break;
int i = vprintf(fmt, av);
va_end(av);
return i;
static inline long int gettimeval(struct timeval *tv)
gettimeofday(tv, NULL);
static inline void statistic_throughput(char *description,
struct timeval *before, struct timeval *after, unsigned long int bytes, long int npkg
)
// printf("\\t -- before time: %ld, %ld\\n", before->tv_sec, before->tv_usec);
// printf("\\t -- after time: %ld, %ld\\n", after->tv_sec, after->tv_usec);
printf("-- %s: Total %.3lf Mbps, bytes = %ld(bits:%ld), npkg = %ld.\\n",
description?description:"Unknown Description",
8*bytes*1.0/((after->tv_sec-before->tv_sec)*1000000
+after->tv_usec-before->tv_usec), bytes, bytes*8, npkg);
static void setaffinity(long int ncpu)
// long int ncpu = sysconf (_SC_NPROCESSORS_ONLN);
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(ncpu>1?ncpu-1:1, &cpuset);
int ret = sched_setaffinity(getpid(), sizeof(cpuset), &cpuset);
log_warn("setaffinity ret = %d\\n", ret);
int j;
for(j=0;j<CPU_SETSIZE; j++)
if(CPU_ISSET(j, &cpuset))
printf("CPU_SETSIZE = %d, j = %d, cpuset = %d\\n", CPU_SETSIZE, j, cpuset);
CPU_CLR(j, &cpuset);
printf("CPU_SETSIZE = %d, j = %d, cpuset = %d\\n", CPU_SETSIZE, j, cpuset);
ret = sched_getaffinity(getpid(), sizeof(cpuset), &cpuset);
for(j=0;j<CPU_SETSIZE; j++)
if(CPU_ISSET(j, &cpuset))
printf("CPU_SETSIZE = %d, j = %d, cpuset = %d\\n", CPU_SETSIZE, j, cpuset);
#endif /*__COMMON_H*/
Makefile
TOPDIR=..
ifeq ($(FF_PATH),)
FF_PATH=$TOPDIR
endif
ifeq ($(FF_DPDK),)
FF_DPDK=$TOPDIR/dpdk/x86_64-native-linuxapp-gcc
endif
LIBS+= -L$FF_PATH/lib -Wl,--whole-archive,-lfstack,--no-whole-archive
LIBS+= -L$FF_DPDK/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive
LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma
all:
cc -O -gdwarf-2 -I../lib -o server server.c $LIBS
.PHONY: clean
clean:
rm -f *.o server client
配置文件server.ini
# F-Stack 参数配置 <rongtao@sylincom.com>
[dpdk]
# Hexadecimal bitmask of cores to run on.
# 绑核
lcore_mask=3
# Number of memory channels.
channel=8
# Specify base virtual address to map.
#base_virtaddr=0x7f0000000000
# Promiscuous mode of nic, defualt: enabled.
# 网卡的混杂模式
promiscuous=1
numa_on=1
# TX checksum offload skip, default: disabled.
# We need this switch enabled in the following cases:
# -> The application want to enforce wrong checksum for testing purposes
# -> Some cards advertize the offload capability. However, doesn't calculate checksum.
tx_csum_offoad_skip=0
# TCP segment offload, default: disabled.
tso=0
# HW vlan strip, default: enabled.
vlan_strip=1
# sleep when no pkts incomming
# unit: microseconds
idle_sleep=0
# sent packet delay time(0-100) while send less than 32 pkts.
# default 100 us.
# if set 0, means send pkts immediately.
# if set >100, will dealy 100 us.
# unit: microseconds
pkt_tx_delay=100
# enabled port list
#
# EBNF grammar:
#
# exp ::= num_list "," num_list
# num_list ::= <num> | <range>
# range ::= <num>"-"<num>
# num ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
#
# examples
# 0-3 ports 0, 1,2,3 are enabled
# 1-3,4,7 ports 1,2,3,4,7 are enabled
#
# If use bonding, shoule config the bonding port id in port_list
# and not config slave port id in port_list
# such as, port 0 and port 1 trank to a bonding port 2,
# should set `port_list=2` and config `[port2]` section
port_list=0,1
# Number of vdev.
nb_vdev=0
# Number of bond.
nb_bond=0
# Each core write into own pcap file, which is open one time, close one time if enough.
# Support dump the first snaplen bytes of each packet.
# if pcap file is lager than savelen bytes, it will be closed and next file was dumped into.
[pcap]
enable = 0
snaplen= 16777216
savelen= 16777216
# Port config section
# Correspond to dpdk.port_list's index: port0, port1...
[port0]
addr=10.170.7.169
netmask=255.255.255.0
broadcast=10.170.7.255
gateway=10.170.7.254
[port1]
addr=10.170.7.170
netmask=255.255.255.0
broadcast=10.170.7.255
gateway=10.170.7.254
# lcore list used to handle this port
# the format is same as port_list
lcore_list=0,1
# bonding slave port list used to handle this port
# need to config while this port is a bonding port
# the format is same as port_list
#slave_port_list=0,1
# Packet capture path, this will hurt performance
#pcap=./a.pcap
# Vdev config section
# orrespond to dpdk.nb_vdev's index: vdev0, vdev1...
# iface : Shouldn't set always.
# path : The vuser device path in container. Required.
# queues : The max queues of vuser. Optional, default 1, greater or equal to the number of processes.
# queue_size : Queue size.Optional, default 256.
# mac : The mac address of vuser. Optional, default random, if vhost use phy NIC, it should be set to the phy NIC's mac.
# cq : Optional, if queues = 1, default 0; if queues > 1 default 1.
#[vdev0]
##iface=/usr/local/var/run/openvswitch/vhost-user0
#path=/var/run/openvswitch/vhost-user0
#queues=1
#queue_size=256
#mac=00:00:00:00:00:01
#cq=0
# bond config section
# See http://doc.dpdk.org/guides/prog_guide/link_bonding_poll_mode_drv_lib.html
#[bond0]
#mode=4
#slave=0000:0a:00.0,slave=0000:0a:00.1
#primary=0000:0a:00.0
#mac=f0:98:38:xx:xx:xx
## opt argument
#socket_id=0
#xmit_policy=l23
#lsc_poll_period_ms=100
#up_delay=10
#down_delay=50
# Kni config: if enabled and method=reject,
# all packets that do not belong to the following tcp_port and udp_port
# will transmit to kernel; if method=accept, all packets that belong to
# the following tcp_port and udp_port will transmit to kernel.
[kni]
enable=1
method=reject
# The format is same as port_list
tcp_port=1-65535
udp_port=1-65535
# FreeBSD network performance tuning configurations.
# Most native FreeBSD configurations are supported.
[freebsd.boot]
hz=100
# Block out a range of descriptors to avoid overlap
# with the kernel's descriptor space.
# You can increase this value according to your app.
fd_reserve=1024
kern.ipc.maxsockets=262144
net.inet.tcp.syncache.hashsize=4096
net.inet.tcp.syncache.bucketlimit=100
net.inet.tcp.tcbhashsize=65536
kern.ncallout=262144
kern.features.inet6=1
net.inet6.ip6.auto_linklocal=1
net.inet6.ip6.accept_rtadv=2
net.inet6.icmp6.rediraccept=1
net.inet6.ip6.forwarding=0
[freebsd.sysctl]
kern.ipc.somaxconn=32768
kern.ipc.maxsockbuf=16777216
net.link.ether.inet.maxhold=5
net.inet.tcp.fast_finwait2_recycle=1
net.inet.tcp.sendspace=16384
net.inet.tcp.recvspace=8192
#net.inet.tcp.nolocaltimewait=1
net.inet.tcp.cc.algorithm=cubic
net.inet.tcp.sendbuf_max=16777216
net.inet.tcp.recvbuf_max=16777216
net.inet.tcp.sendbuf_auto=1
net.inet.tcp.recvbuf_auto=1
net.inet.tcp.sendbuf_inc=16384
net.inet.tcp.recvbuf_inc=524288
net.inet.tcp.sack.enable=1
net.inet.tcp.blackhole=1
net.inet.tcp.msl=2000
net.inet.tcp.delayed_ack=0
net.inet.udp.blackhole=1
net.inet.ip.redirect=0
net.inet.ip.forwarding=0
客户端代码
client.c
/**
* 测试F-Stack的UDP发包速率
* 作者：荣涛 <rongtao@sylincom.com>
* 时间：2020年7月16日10:08:34
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <netinet/ip.h>
#include "common.h"
int main(int argc, char *argv[])
int sockfd, n;
struct sockaddr_in servaddr;
socklen_t servlen = sizeof(servaddr);
struct timeval tvbrfore, tvafter;
long int npkg = 0, nbyte=0;
char sendline[MAXLINE], recvline[MAXLINE + 1];
if(argc != 2)
perror("Usage: udpcli <IPAddress>");
exit(1);
sockfd = udpsocket_client(argv[1], &servaddr);
printf("Client Input:");
while(fgets(sendline, MAXLINE, stdin) != NULL)
if(sendto(sockfd, sendline, strlen(sendline), 0,
(struct sockaddr *)&servaddr, servlen) < 0)
perror("sendto error");
exit(1);
else
printf("Client send %d: %s", sockfd, sendline);
npkg = 0, nbyte=0;
memset(&tvbrfore, 0, sizeof(struct timeval));
memset(&tvafter, 0, sizeof(struct timeval));
/* 统计时间 */
my_gettimeval(&tvbrfore);
while(1)
if((n = recvfrom(sockfd, recvline, MAXLINE, 0, NULL, &servlen)) < 0)
perror("recvfrom error");
break;
else
// printf("Client recv %d: %s", sockfd, recvline);
nbyte += n;
if(npkg++ > 10000)
break;
recvline[n] = '\\0';
/* 统计时间 */
my_gettimeval(&tvafter);
/* 输出此段时间内的速率 */
my_statistic_throughput("Recvfrom", &tvbrfore, &tvafter, nbyte, npkg-1);
/* 准备下次触发f_stack服务端 进行发包测试 */
printf(">>");
return 1;
common.h
/**
* 测试F-Stack的UDP发包速率
* 作者：荣涛 <rongtao@sylincom.com>
* 时间：2020年7月16日10:08:34
*/
#ifndef _COMMON_H
#define _COMMON_H
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/sysinfo.h>
#include <time.h>
#include <stdint.h>
#define PORT 2152
#define MAXLINE 1500
int udpsocket_server();
int udpsocket_client(const char *ipv4, struct sockaddr_in *servaddr);
inline long int my_gettimeval(struct timeval *tv);
inline void my_statistic_throughput(char *description,
struct timeval *before, struct timeval *after, unsigned long int bytes, long int npkg);
#endif /*<_COMMON_H>*/
common.c
/**
* 测试F-Stack的UDP发包速率
* 作者：荣涛 <rongtao@sylincom.com>
* 时间：2020年7月16日10:08:34
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <string.h>
#include <sys/types.h>
#include <netinet/ip.h>
#include "common.h"
int udpsocket_server()
int sockfd;
struct sockaddr_in servaddr;
bzero(&servaddr, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
servaddr.sin_port = htons(PORT);
if((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
return -1;
if(bind(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)))
return -1;
return sockfd;
int udpsocket_client(const char *ipv4, struct sockaddr_in *servaddr)
int sockfd, t;
bzero(servaddr, sizeof(struct sockaddr_in));
servaddr->sin_family = AF_INET;
servaddr->sin_port = htons(PORT);
if((t = inet_pton(AF_INET, ipv4, &servaddr->sin_addr)) <= 0)
return -1;
if((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
return -1;
return sockfd;
inline long int my_gettimeval(struct timeval *tv)
gettimeofday(tv, NULL);
inline void my_statistic_throughput(char *description,
struct timeval *before, struct timeval *after, unsigned long int bytes, long int npkg)
// printf("\\t -- before time: %ld, %ld\\n", before->tv_sec, before->tv_usec);
// printf("\\t -- after time: %ld, %ld\\n", after->tv_sec, after->tv_usec);
printf("-- %s: Total %.3lf Mbps, bytes = %ld(bits:%ld), npkg = %ld.\\n",
description?description:"Unknown Description",
8*bytes*1.0/((after->tv_sec-before->tv_sec)*1000000
+after->tv_usec-before->tv_usec), bytes, bytes*8, npkg);
Makefile
ALL:
gcc client.c common.c -o client -lm
clean:
rm *~
运行结果
以上是关于F-Stack实现UDP服务端客户端，并进行吞吐量测试的实现的主要内容，如果未能解决你的问题，请参考以下文章