Testing ovn manually based on LXD (by quqi99)
Posted quqi99
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Testing ovn manually based on LXD (by quqi99)相关的知识，希望对你有一定的参考价值。
作者：张华 发表于：2022-05-27
版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明
准备两个LXD容器
$ lxc list
+--------+---------+-----------------------+------+-----------+-----------+
| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
+--------+---------+-----------------------+------+-----------+-----------+
| master | RUNNING | 192.168.122.20 (ens3) | | CONTAINER | 0 |
+--------+---------+-----------------------+------+-----------+-----------+
| node1 | RUNNING | 192.168.122.21 (ens3) | | CONTAINER | 0 |
+--------+---------+-----------------------+------+-----------+-----------+
步骤如下:
# install lxd, refer - https://blog.csdn.net/quqi99/article/details/52131486
sudo snap install lxd --classic
sudo usermod -aG $USER lxd
# MUST NOT use sudo, so must cd to home dir to run it
cd ~ && lxd init --auto
sudo chown -R $USER ~/.config/
export EDITOR=vim
# use static subnet 192.168.122.0/24 (qemu also uses this subnet) for lxd as well
sudo virsh net-destroy default
lxc network show lxdbr0
lxc network set lxdbr0 ipv4.address=192.168.122.1/24
lxc network set lxdbr0 ipv6.address none
ip addr show lxdbr0
sudo iptables-save |grep 192.168.122
ps -ef |grep 192.168.122
# set lxc profile - https://github.com/openstack-charmers/openstack-on-lxd.git
cat << EOF | tee ./lxd-profile.yaml
config:
boot.autostart: "true"
linux.kernel_modules: openvswitch,nbd,ip_tables,ip6_tables
security.nesting: "true"
security.privileged: "true"
description: ""
devices:
ens3:
mtu: "9000"
name: ens3
nictype: bridged
parent: lxdbr0
type: nic
ens8:
mtu: "9000"
name: ens8
nictype: bridged
parent: lxdbr0
type: nic
kvm:
path: /dev/kvm
type: unix-char
mem:
path: /dev/mem
type: unix-char
root:
path: /
pool: default
type: disk
tun:
path: /dev/net/tun
type: unix-char
name: juju-default
used_by: []
EOF
lxc profile create juju-default 2>/dev/null || echo "juju-default profile already exists"
cat ./lxd-profile.yaml |lxc profile edit juju-default
#lxc profile device set juju-default root pool=default
lxc profile show juju-default
# create two test lxd containers
lxc network show lxdbr0
cat << EOF | tee network.yml
version: 1
config:
- type: physical
name: ens3
subnets:
- type: static
ipv4: true
address: 192.168.122.20
netmask: 255.255.255.0
gateway: 192.168.122.1
control: auto
- type: nameserver
address: 8.8.8.8
EOF
lxc launch ubuntu:focal master -p juju-default --config=user.network-config="$(cat network.yml)"
cat << EOF | tee network.yml
version: 1
config:
- type: physical
name: ens3
subnets:
- type: static
ipv4: true
address: 192.168.122.21
netmask: 255.255.255.0
gateway: 192.168.122.1
control: auto
- type: nameserver
address: 192.168.99.1
EOF
lxc launch ubuntu:focal node1 -p juju-default --config=user.network-config="$(cat network.yml)"
lxc exec `lxc list |grep master |awk -F '|' 'print $2'` bash
lxc exec `lxc list |grep node1 |awk -F '|' 'print $2'` bash
安装OVN
master节点将同时作为控制面与数据面, 所以除了安装ovn-central, 也安装了ovn-host与openvswitch-switch
#on master,
lxc exec `lxc list |grep master |awk -F '|' 'print $2'` bash
apt install ovn-central openvswitch-switch ovn-host net-tools -y
ovn-nbctl set-connection ptcp:6641
ovn-sbctl set-connection ptcp:6642
netstat -lntp |grep 664
#on node1 and master
lxc exec `lxc list |grep node1 |awk -F '|' 'print $2'` bash
apt install openvswitch-switch ovn-host net-tools -y
ovs-vsctl add-br br-int
ovs-vsctl set bridge br-int protocols=OpenFlow10,OpenFlow11,OpenFlow12,OpenFlow13,OpenFlow14,OpenFlow15
#on all compute nodes (both master and node1), make ovn-controller connect to southbound db
ovs-vsctl set open_vswitch . \\
external_ids:ovn-remote=tcp:192.168.122.20:6642 \\
external_ids:ovn-encap-ip=$(ip addr show ens3| awk '$1 == "inet" print $2' | cut -f1 -d/) \\
external_ids:ovn-encap-type=geneve \\
external_ids:system-id=$(hostname)
逻辑拓扑, 物理拓扑及实现
# create vRouer (lr1) and two vSwitch (ls1 and ls2)
ovn-nbctl ls-add ls1
ovn-nbctl ls-add ls2
ovn-nbctl lr-add lr1
# dhcp for ls1
#ovn-nbctl set logical_switch ls1 other_config:subnet="10.10.10.0/24" other_config:exclude_ips="10.10.10.2..10.10.10.10"
#ovn-nbctl dhcp-options-create 10.10.10.0/24
#DHCP_UUID=$(ovn-nbctl --bare --columns=_uuid find dhcp_options cidr="10.10.10.0/24")
#ovn-nbctl dhcp-options-set-options $DHCP_UUID lease_time=3600 router=10.10.10.1 server_id=10.10.10.1 server_mac=c0:ff:ee:00:00:01
#ovn-nbctl list dhcp_options
# connect ls1 to lr1
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 10.10.10.1/24
ovn-nbctl lsp-add ls1 ls1-lr1
ovn-nbctl lsp-set-type ls1-lr1 router
ovn-nbctl lsp-set-addresses ls1-lr1 00:00:00:00:00:01
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1
# connect ls2 to lr1
ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 10.10.20.1/24
ovn-nbctl lsp-add ls2 ls2-lr1
ovn-nbctl lsp-set-type ls2-lr1 router
ovn-nbctl lsp-set-addresses ls2-lr1 00:00:00:00:00:02
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2
# create 4 test VMs on two chassises
ovn-nbctl lsp-add ls1 ls1-vm1
ovn-nbctl lsp-set-addresses ls1-vm1 "00:00:00:00:00:03 10.10.10.2"
ovn-nbctl lsp-set-port-security ls1-vm1 "00:00:00:00:00:03 10.10.10.2"
ovn-nbctl lsp-add ls1 ls1-vm2
ovn-nbctl lsp-set-addresses ls1-vm2 "00:00:00:00:00:04 10.10.10.3"
ovn-nbctl lsp-set-port-security ls1-vm2 "00:00:00:00:00:04 10.10.10.3"
ovn-nbctl lsp-add ls2 ls2-vm1
ovn-nbctl lsp-set-addresses ls2-vm1 "00:00:00:00:00:03 10.10.20.2"
ovn-nbctl lsp-set-port-security ls2-vm1 "00:00:00:00:00:03 10.10.20.2"
ovn-nbctl lsp-add ls2 ls2-vm2
ovn-nbctl lsp-set-addresses ls2-vm2 "00:00:00:00:00:04 10.10.20.3"
ovn-nbctl lsp-set-port-security ls2-vm2 "00:00:00:00:00:04 10.10.20.3"
# on master
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 netns vm1
ip netns exec vm1 ip link set vm1 address 00:00:00:00:00:03
ip netns exec vm1 ip addr add 10.10.10.2/24 dev vm1
ip netns exec vm1 ip link set vm1 up
ip netns exec vm1 ip route add default via 10.10.10.1 dev vm1
ovs-vsctl set Interface vm1 external_ids:iface-id=ls1-vm1
ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 netns vm2
ip netns exec vm2 ip link set vm2 address 00:00:00:00:00:04
ip netns exec vm2 ip addr add 10.10.10.3/24 dev vm2
ip netns exec vm2 ip link set vm2 up
ip netns exec vm2 ip route add default via 10.10.10.1 dev vm2
ovs-vsctl set Interface vm2 external_ids:iface-id=ls1-vm2
# on node1
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 netns vm1
ip netns exec vm1 ip link set vm1 address 00:00:00:00:00:03
ip netns exec vm1 ip addr add 10.10.20.2/24 dev vm1
ip netns exec vm1 ip link set vm1 up
ip netns exec vm1 ip route add default via 10.10.20.1 dev vm1
ovs-vsctl set Interface vm1 external_ids:iface-id=ls2-vm1
ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 netns vm2
ip netns exec vm2 ip link set vm2 address 00:00:00:00:00:04
ip netns exec vm2 ip addr add 10.10.20.3/24 dev vm2
ip netns exec vm2 ip link set vm2 up
ip netns exec vm2 ip route add default via 10.10.20.1 dev vm2
ovs-vsctl set Interface vm2 external_ids:iface-id=ls2-vm2
# create the static route
ovn-nbctl lr-route-add lr1 "0.0.0.0/0" 10.10.40.1
# create ha-chassis-group
ovn-nbctl lrp-add lr1 lr1-lslocal 00:00:00:00:00:05 10.10.40.1/24
ovn-nbctl ha-chassis-group-add ha1
ovn-nbctl ha-chassis-group-add-chassis ha1 master 1
ovn-nbctl ha-chassis-group-add-chassis ha1 node1 2
ha1_uuid=`ovn-nbctl --bare --columns _uuid find ha_chassis_group name="ha1"`
ovn-nbctl set Logical_Router_Port lr1-lslocal ha_chassis_group=$ha1_uuid
# connect lslocal to lr1
ovn-nbctl ls-add lslocal
ovn-nbctl lsp-add lslocal lslocal-lr1
ovn-nbctl lsp-set-type lslocal-lr1 router
ovn-nbctl lsp-set-addresses lslocal-lr1 00:00:00:00:00:05
ovn-nbctl lsp-set-options lslocal-lr1 router-port=lr1-lslocal
ovn-nbctl lsp-add lslocal lslocal-localnet
ovn-nbctl lsp-set-addresses lslocal-localnet unknown
ovn-nbctl lsp-set-type lslocal-localnet localnet
ovn-nbctl lsp-set-options lslocal-localnet network_name=externalnet
# on master
ovs-vsctl add-br br-ens8
ovs-vsctl add-port br-ens8 ens8
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalnet:br-ens8
#ovn-nbctl lrp-set-gateway-chassis lr1-lslocal master 1
#ovn-nbctl lrp-set-gateway-chassis lr1-lslocal node1 2
ovs-vsctl set Open_vSwitch . external-ids:ovn-cms-options=\\"enable-chassis-as-gw\\"
ip link set dev br-ens8 up
ip addr add 10.10.40.2/24 dev br-ens8
ovs-vsctl get Open_vSwitch . external-ids
# on node1
ovs-vsctl add-br br-ens8
ovs-vsctl add-port br-ens8 ens8
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalnet:br-ens8
#ovn-nbctl lrp-set-gateway-chassis lr1-lslocal master 1
#ovn-nbctl lrp-set-gateway-chassis lr1-lslocal node1 2
ovs-vsctl set Open_vSwitch . external-ids:ovn-cms-options=\\"enable-chassis-as-gw\\"
ip link set dev br-ens8 up
ip addr add 10.10.40.3/24 dev br-ens8
# add NAT item
ovn-nbctl -- --id=@nat create nat type="snat" logical_ip=10.10.10.0/24 \\
external_ip=10.10.40.1 -- add logical_router lr1 nat @nat
# query ways
ovn-nbctl list gateway_chassis
ovn-nbctl show
ovn-nbctl list logical_router
ovn-nbctl list logical_router_port
ovn-nbctl list logical_switch
ovn-nbctl list logical_switch_port
ovn-nbctl list ha_chassis_group
ovn-nbctl -f csv list ha_chassis |egrep -v '^_uuid' |sort -t ',' -k 4
ovn-sbctl show
ovn-sbctl list Port_Binding
ovs-vsctl show
测试结果
# Test
root@master:~# ip netns exec vm1 ping 10.10.20.3 -c1
PING 10.10.20.3 (10.10.20.3) 56(84) bytes of data.
64 bytes from 10.10.20.3: icmp_seq=1 ttl=63 time=0.146 ms
root@master:~# ip netns exec vm1 ping 10.10.40.1 -c1
PING 10.10.40.1 (10.10.40.1) 56(84) bytes of data.
64 bytes from 10.10.40.1: icmp_seq=1 ttl=254 time=0.276 ms
一些输出
root@master:~# ovn-nbctl show
switch 31f0f2f4-1a02-4efd-8731-09993279f917 (lslocal)
port lslocal-localnet
type: localnet
addresses: ["unknown"]
port lslocal-lr1
type: router
addresses: ["00:00:00:00:00:05"]
router-port: lr1-lslocal
switch fae87181-383e-41eb-8c41-5a6b52c358ca (ls1)
port ls1-vm2
addresses: ["00:00:00:00:00:04 10.10.10.3"]
port ls1-lr1
type: router
addresses: ["00:00:00:00:00:01"]
router-port: lr1-ls1
port ls1-vm1
addresses: ["00:00:00:00:00:03 10.10.10.2"]
switch bf707a77-f6a0-4bd6-9549-fb3027a4b539 (ls2)
port ls2-lr1
type: router
addresses: ["00:00:00:00:00:02"]
router-port: lr1-ls2
port ls2-vm1
addresses: ["00:00:00:00:00:03 10.10.20.2"]
port ls2-vm2
addresses: ["00:00:00:00:00:04 10.10.20.3"]
router d872c966-89f7-46db-b5dd-362315042b35 (lr1)
port lr1-ls2
mac: "00:00:00:00:00:02"
networks: ["10.10.20.1/24"]
port lr1-ls1
mac: "00:00:00:00:00:01"
networks: ["10.10.10.1/24"]
port lr1-lslocal
mac: "00:00:00:00:00:05"
networks: ["10.10.40.1/24"]
gateway chassis: [node1 master]
nat 62e5b090-b6a7-416c-8f8c-a4c4f9290236
external ip: "10.10.40.1"
logical ip: "10.10.10.0/24"
type: "snat"
root@master:~# ovn-sbctl show
Chassis master
hostname: master
Encap geneve
ip: "192.168.122.20"
options: csum="true"
Port_Binding ls1-vm1
Port_Binding ls1-vm2
Chassis node1
hostname: node1
Encap geneve
ip: "192.168.122.21"
options: csum="true"
Port_Binding cr-lr1-lslocal
Port_Binding ls2-vm2
Port_Binding ls2-vm1
root@master:~# ovs-vsctl show
7fa21184-4091-4c56-bf22-f27bd43b049d
Bridge br-ens8
Port br-ens8
Interface br-ens8
type: internal
Port ens8
Interface ens8
Port patch-lslocal-localnet-to-br-int
Interface patch-lslocal-localnet-to-br-int
type: patch
options: peer=patch-br-int-to-lslocal-localnet
Bridge br-int
Port vm2
Interface vm2
type: internal
Port ovn-node1-0
Interface ovn-node1-0
type: geneve
options: csum="true", key=flow, remote_ip="192.168.122.21"
bfd_status: diagnostic="No Diagnostic", flap_count="1", forwarding="true", remote_diagnostic="No Diagnostic", remote_state=up, state=up
Port br-int
Interface br-int
type: internal
Port vm1
Interface vm1
type: internal
Port patch-br-int-to-lslocal-localnet
Interface patch-br-int-to-lslocal-localnet
type: patch
options: peer=patch-lslocal-localnet-to-br-int
ovs_version: "2.13.5"
root@node1:~# ovs-vsctl show
1f40614f-ea1f-40c5-b564-97e7d4a678e6
Bridge br-ens8
Port br-ens8
Interface br-ens8
type: internal
Port ens8
Interface ens8
Port patch-lslocal-localnet-to-br-int
Interface patch-lslocal-localnet-to-br-int
type: patch
options: peer=patch-br-int-to-lslocal-localnet
Bridge br-int
Port vm1
Interface vm1
type: internal
Port patch-br-int-to-lslocal-localnet
Interface patch-br-int-to-lslocal-localnet
type: patch
options: peer=patch-lslocal-localnet-to-br-int
Port ovn-master-0
Interface ovn-master-0
type: geneve
options: csum="true", key=flow, remote_ip="192.168.122.20"
bfd_status: diagnostic="No Diagnostic", flap_count="1", forwarding="true", remote_diagnostic="No Diagnostic", remote_state=up, state=up
Port br-int
Interface br-int
type: internal
Port vm2
Interface vm2
type: internal
ovs_version: "2.13.5"
root@master:~# ovn-nbctl list ha_chassis_group
_uuid : c843f65f-a215-4bd7-8427-c2da6ec33cf8
external_ids :
ha_chassis : [25448727-ce5f-4676-974e-ef7d3e1ee915, d236dd68-c65b-4842-8e8d-b999d6895e09]
name : ha1
root@master:~# ovn-nbctl -f csv list ha_chassis |egrep -v '^_uuid' |sort -t ',' -k 4
25448727-ce5f-4676-974e-ef7d3e1ee915,master,,1
d236dd68-c65b-4842-8e8d-b999d6895e09,node1,,2
root@master:~# ovn-nbctl list gateway_chassis
_uuid : 3d670f79-fe01-432c-93d1-113aa5747fcc
chassis_name : master
external_ids :
name : lr1-lslocal-master
options :
priority : 1
_uuid : 0b6cc1ab-bb03-4674-81c9-1257bcfcbd7c
chassis_name : node1
external_ids :
name : lr1-lslocal-node1
options :
priority : 2
root@master:~# ovs-vsctl get Open_vSwitch . external-ids
hostname=master, ovn-bridge-mappings="externalnet:br-ens8", ovn-cms-options=enable-chassis-as-gw, ovn-encap-ip="192.168.122.20", ovn-encap-type=geneve, ovn-remote="tcp:192.168.122.20:6642", rundir="/var/run/openvswitch", system-id=master
root@node1:~# ovs-vsctl get Open_vSwitch . external-ids
hostname=node1, ovn-bridge-mappings="externalnet:br-ens8", ovn-cms-options=enable-chassis-as-gw, ovn-encap-ip="192.168.122.21", ovn-encap-type=geneve, ovn-remote="tcp:192.168.122.20:6642", rundir="/var/run/openvswitch", system-id=node1
reference
[1] ovn通过分布式网关端口连接外部网络 - https://www.jianshu.com/p/dc565d6aaebd
以上是关于Testing ovn manually based on LXD (by quqi99)的主要内容，如果未能解决你的问题，请参考以下文章
Testing ovn manually based on LXD (by quqi99)
OVN Base flow L2/L3 forwarding
set up ovn based sr-iov test env (by quqi99)
set up ovn based sr-iov test env (by quqi99)