虚拟化网络之QEMU-KVM虚拟机模拟net-space功能

Posted 2021-05-02 运维扫盲人

环境：

[root@rs-1 ~]# cat /etc/redhat-release 

CentOS Linux release 7.7.1908 (Core)

• 安装QEMU-KVM虚拟机管理工具
  

[root@rs-1 ~]# yum groupinfo "Virtualization Host"[root@rs-1 ~]# yum install -y qemu-kvm[root@rs-1 ~]# ln -sv /usr/libexec/qemu-kvm  /usr/bin/

• 加载KVM模块
  

[root@rs-1 ~]# modprobe kvm[root@rs-1 ~]# lsmod | grep kvmkvm_intel 188688 4 kvm 636965 1 kvm_intelirqbypass 13503 1 kvm

• 下载cirros镜像

[root@rs-1 ~]# wget https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img

• 创建连接虚拟机与netns的桥设备并激活
  

[root@rs-1 ~]# yum install -y bridge-utils-1.5-9.el7.x86_64[root@rs-1 ~]# brctl addbr br-in[root@rs-1 ~]# brctl showbridge name bridge id STP enabled interfacesbr-in 8000.a23b87955f31 no[root@rs-1 ~]# ip link set dev br-in up[root@rs-1 ~]# ifconfig br-inbr-in: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet6 fe80::285c:5eff:fe8c:fd98 prefixlen 64 scopeid 0x20<link> ether 7e:95:ec:b6:cc:25 txqueuelen 1000 (Ethernet) RX packets 8 bytes 1000 (1000.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 8 bytes 656 (656.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

注意：创建虚拟桥和物理桥设备后务必进行激活操作；

• 定义虚拟机启动自动添加至桥设备脚本
  

[root@rs-1 ~]# cat /etc/qemu-ifup #!/bin/bash#Bridge=br-inif [ -n "$1" ]; then ip link set $1 up brctl addif $Bridge $1 [ $? -eq 0 ] && exit 0 || exit 1else  echo "Error: no interface specified." exit 1fi[root@rs-1 ~]# chmod +x /etc/qemu-ifup

• 创建并启动虚拟机
  

[root@rs-1 ~]# wget https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img[root@rs-1 ~]# cp cirros-0.3.0-x86_64-disk.img /images/cirros/test1.qcow2[root@rs-1 ~]# cp cirros-0.3.0-x86_64-disk.img  /images/cirros/test2.qcow2[root@rs-1 ~]# qemu-kvm -m 128 -cpu host -smp 2 -name vm1 -drive file=/images/cirros/test1.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:12:34:57 -net tap,ifname=vif1.0,script=/etc/qemu-ifup --nographic[root@rs-1 ~]# qemu-kvm -m 128 -cpu host -smp 2 -name vm1 -drive file=/images/cirros/test2.qcow2,if=virtio,media=disk -net nic,macaddr=52:54:00:12:34:68 -net tap,ifname=vif2.0,script=/etc/qemu-ifup --nographicuser：cirros password: cubswin:)[root@rs-1 ~]# brctl showbridge name bridge id STP enabled interfacesbr-in        8000.92f5b9bbaf9d  no          vif1.0 vif2.0

• 添加netns作为连接虚拟机桥与物理桥的路由器
  

[root@rs-1 ~]# ip netns add r1[root@rs-1 ~]# ip netns listr1[root@rs-1 ~]# ip netns exec r1 ifconfig -alo: flags=8<LOOPBACK> mtu 65536 loop txqueuelen 1000 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

[root@rs-1 ~]# ip link help add[root@rs-1 ~]# ip link add rinr type veth peer name rins[root@rs-1 ~]# ip link show | grep rin14: rins@rinr: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 100015: rinr@rins: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000[root@rs-1 ~]# ip link set rins up[root@rs-1 ~]# ip link set rinr up[root@rs-1 ~]# brctl addif br-in rins[root@rs-1 ~]# brctl show br-inbridge name bridge id STP enabled interfacesbr-in 8000.72e3d9c9c3af no rins vif1.0                                             vif2.0[root@rs-1 ~]# ip link set rinr netns r1[root@rs-1 ~]# ip netns exec r1 ifconfig -alo: flags=8<LOOPBACK> mtu 65536 loop txqueuelen 1000 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
rinr: flags=4098<BROADCAST,MULTICAST> mtu 1500 ether 56:25:06:8a:8b:8e txqueuelen 1000 (Ethernet) RX packets 8 bytes 656 (656.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 8 bytes 656 (656.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0[root@rs-1 ~]# ip netns exec r1 ip link set rinr name eth0 [root@rs-1 ~]# ip netns exec r1 ifconfig -aeth0: flags=4098<BROADCAST,MULTICAST> mtu 1500 ether 56:25:06:8a:8b:8e txqueuelen 1000 (Ethernet) RX packets 8 bytes 656 (656.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 8 bytes 656 (656.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=8<LOOPBACK> mtu 65536 loop txqueuelen 1000 (Local Loopback) RX packets 0 bytes 0 (0.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 0 bytes 0 (0.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0[root@rs-1 ~]# ip netns exec r1 ifconfig eth0 172.17.0.254/24 up[root@rs-1 ~]# ip netns exec r1 ifconfig eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.17.0.254 netmask 255.255.255.0 broadcast 172.17.0.255 inet6 fe80::5425:6ff:fe8a:8b8e prefixlen 64 scopeid 0x20<link> ether 56:25:06:8a:8b:8e txqueuelen 1000 (Ethernet) RX packets 8 bytes 656 (656.0 B) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 14 bytes 1172 (1.1 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

# ifconfig eth0 172.17.0.1/24 up# ifconfig eth0eth0 Link encap:Ethernet HWaddr 52:54:00:12:34:57  inet addr:172.17.0.1 Bcast:172.17.255.255 Mask:255.255.0.0 inet6 addr: fe80::5054:ff:fe12:3457/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000           RX bytes:0 (0.0 B)  TX bytes:1112 (1.0 KiB) # ifconfig eth0     Link encap:Ethernet  HWaddr 52:54:00:12:34:68   inet addr:172.17.0.2 Bcast:172.17.255.255 Mask:255.255.0.0 inet6 addr: fe80::5054:ff:fe12:3468/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000  RX bytes:0 (0.0 B) TX bytes:1112 (1.0 KiB) # ifconfig eth0eth0 Link encap:Ethernet HWaddr 52:54:00:12:34:57  inet addr:172.17.0.1 Bcast:172.17.255.255 Mask:255.255.0.0 inet6 addr: fe80::5054:ff:fe12:3457/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:18 errors:0 dropped:0 overruns:0 frame:0 TX packets:26 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:1000  RX bytes:1536 (1.5 KiB) TX bytes:2540 (2.4 KiB)
# ping 172.17.0.2PING 172.17.0.2 (172.17.0.2): 56 data bytes64 bytes from 172.17.0.2: seq=0 ttl=64 time=4.030 ms64 bytes from 172.17.0.2: seq=1 ttl=64 time=1.930 ms

• 设置两台虚拟机网关指向netns的eth0
  

vm1：# route add default gw 172.17.0.254 dev eth0vm2：# route add default gw 172.17.0.254 dev eth0# route -nKernel IP routing tableDestination Gateway Genmask Flags Metric Ref Use Iface0.0.0.0 172.17.0.254 0.0.0.0 UG 0 0 0 eth0172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 eth0

• 测试虚拟机可以与netns可通信
  

# ping 172.17.0.254PING 172.17.0.254 (172.17.0.254): 56 data bytes64 bytes from 172.17.0.254: seq=0 ttl=64 time=1.943 ms64 bytes from 172.17.0.254: seq=1 ttl=64 time=1.410 ms64 bytes from 172.17.0.254: seq=2 ttl=64 time=3.915 ms --- 172.17.0.254 ping statistics ---1 packets transmitted, 1 packets received, 0% packet loss

• 添加物理桥并添加物理网卡
  

[root@rs-1 ~]# brctl addbr br-ex[root@rs-1 ~]# cat /etc/sysconfig/network-scripts/ifcfg-br-ex TYPE=BridgeBOOTPROTO=dhcpDEFROUTE=yesNAME=br-exDEVICE=br-exONBOOT=yes[root@rs-1 ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens33TYPE=EthernetDEFROUTE=yesNAME=ens33BRIDGE=br-exDEVICE=ens33ONBOOT=yes

• 创建连接netns与物理网桥的一对网卡

[root@rs-1 ~]# ip link add rexr type veth peer name rexs [root@rs-1 ~]# brctl addif br-ex rexs[root@rs-1 ~]# ip link set rexs up   [root@rs-1 ~]# brctl showbridge name bridge id STP enabled interfacesbr-ex        8000.000c29320ee2  no     ens33                                             rexsbr-in 8000.7e95ecb6cc25 no rins vif1.0 vif2.0[root@rs-1 ~]# ip link set rexr netns r1[root@rs-1 ~]# ip netns exec r1 ip link set rexr name eth1[root@rs-1 ~]# ip netns exec r1 ip link set eth1 up[root@rs-1 ~]# ip netns exec r1 ifconfig eth1 192.168.1.77/24 up[root@rs-1 ~]# ip netns exec r1 ifconfigeth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.17.0.254 netmask 255.255.0.0 broadcast 172.17.255.255 inet6 fe80::9809:ecff:fee4:ec6e prefixlen 64 scopeid 0x20<link> ether 9a:09:ec:e4:ec:6e txqueuelen 1000 (Ethernet) RX packets 106 bytes 9644 (9.4 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 44 bytes 3720 (3.6 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 192.168.1.77 netmask 255.255.255.0 broadcast 192.168.1.255 inet6 fe80::d046:e7ff:feb5:e1de prefixlen 64 scopeid 0x20<link> ether d2:46:e7:b5:e1:de txqueuelen 1000 (Ethernet) RX packets 91 bytes 14064 (13.7 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 8 bytes 656 (656.0 B) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0虚拟机测试与netns的eth1是否可通# ping 192.168.1.77PING 192.168.1.77 (192.168.1.77): 56 data bytes64 bytes from 192.168.1.77: seq=0 ttl=64 time=0.648 ms64 bytes from 192.168.1.77: seq=1 ttl=64 time=0.941 ms64 bytes from 192.168.1.77: seq=2 ttl=64 time=1.077 ms64 bytes from 192.168.1.77: seq=3 ttl=64 time=1.675 ms测试虚拟机与物理桥br-ex不能通信并抓包分析# ping 192.168.1.107[root@rs-1 ~]# tcpdump -i vif1.0 -nn icmptcpdump: verbose output suppressed, use -v or -vv for full protocol decodelistening on vif1.0, link-type EN10MB (Ethernet), capture size 262144 bytes22:17:58.638638 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 32, length 6422:17:59.640113 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 33, length 6422:18:00.642063 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 34, length 6422:18:01.645045 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 35, length 64[root@rs-1 ~]# tcpdump -i br-ex -nn icmptcpdump: verbose output suppressed, use -v or -vv for full protocol decodelistening on br-ex, link-type EN10MB (Ethernet), capture size 262144 bytes22:19:51.302720 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 144, length 6422:19:52.304090 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 145, length 6422:19:53.307207 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 1793, seq 146, length 64

• netns添加SNAT规则并且开启核心转发再测试虚拟机与物理网桥是否可通

[root@rs-1 ~]# ip netns exec r1 iptables -t nat -A POSTROUTING -s 172.17.0.0/24 ! -d 172.17.0.0/24 -j SNAT --to-source 192.168.1.77 [root@rs-1 ~]# ip netns exec r1 iptables -t nat -vnL# ping 192.168.1.107PING 192.168.1.107 (192.168.1.107): 56 data bytes64 bytes from 192.168.1.107: seq=0 ttl=63 time=1.247 ms64 bytes from 192.168.1.107: seq=1 ttl=63 time=1.112 ms64 bytes from 192.168.1.107: seq=2 ttl=63 time=1.363 ms[root@rs-1 ~]# tcpdump -i vif1.0 -nn icmptcpdump: verbose output suppressed, use -v or -vv for full protocol decodelistening on vif1.0, link-type EN10MB (Ethernet), capture size 262144 bytes22:25:15.102045 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 2561, seq 9, length 6422:25:15.102170 IP 192.168.1.107 > 172.17.0.1: ICMP echo reply, id 2561, seq 9, length 6422:25:16.102492 IP 172.17.0.1 > 192.168.1.107: ICMP echo request, id 2561, seq 10, length 6422:25:16.102601 IP 192.168.1.107 > 172.17.0.1: ICMP echo reply, id 2561, seq 10, length 64[root@rs-1 ~]# tcpdump -i br-ex -nn icmptcpdump: verbose output suppressed, use -v or -vv for full protocol decodelistening on br-ex, link-type EN10MB (Ethernet), capture size 262144 bytes22:25:23.125660 IP 192.168.1.77 > 192.168.1.107: ICMP echo request, id 2561, seq 17, length 6422:25:23.125708 IP 192.168.1.107 > 192.168.1.77: ICMP echo reply, id 2561, seq 17, length 6422:25:24.127291 IP 192.168.1.77 > 192.168.1.107: ICMP echo request, id 2561, seq 18, length 6422:25:24.127333 IP 192.168.1.107 > 192.168.1.77: ICMP echo reply, id 2561, seq 18, length 64测试虚拟机与其他同网段宿主机是否可通信：# ping 192.168.1.104PING 192.168.1.104 (192.168.1.104): 56 data bytes64 bytes from 192.168.1.104: seq=0 ttl=63 time=26.191 ms64 bytes from 192.168.1.104: seq=1 ttl=63 time=1.904 ms64 bytes from 192.168.1.104: seq=2 ttl=63 time=2.001 ms[root@rs-1 ~]# tcpdump -i br-ex -nn icmptcpdump: verbose output suppressed, use -v or -vv for full protocol decodelistening on br-ex, link-type EN10MB (Ethernet), capture size 262144 bytes22:39:52.299854 IP 192.168.1.77 > 192.168.1.104: ICMP echo request, id 2817, seq 17, length 6422:39:52.300300 IP 192.168.1.104 > 192.168.1.77: ICMP echo reply, id 2817, seq 17, length 6422:39:53.304345 IP 192.168.1.77 > 192.168.1.104: ICMP echo request, id 2817, seq 18, length 6422:39:53.304658 IP 192.168.1.104 > 192.168.1.77: ICMP echo reply, id 2817, seq 18, length 6422:39:54.309295 IP 192.168.1.77 > 192.168.1.104: ICMP echo request, id 2817, seq 19, length 6422:39:54.309937 IP 192.168.1.104 > 192.168.1.77: ICMP echo reply, id 2817, seq 19, length 6422:39:55.312286 IP 192.168.1.77 > 192.168.1.104: ICMP echo request, id 2817, seq 20, length 6422:39:55.312778 IP 192.168.1.104 > 192.168.1.77: ICMP echo reply, id 2817, seq 20, length 64

• 开启宿主机核心转发功能
  

[root@rs-1 ~]# echo net.ipv4.ip_forward = 1 >> /etc/sysctl.conf [root@rs-1 ~]# sysctl -p

注意：在测试过程中务必开启宿主机的核心转发功能；使用QEMU-KVM虚拟机时需要开启Intel VT-x/EPT的硬件支持的虚拟化功能；