Try kolla-ansible (by quqi99)

Posted 2021-08-17 quqi99

版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明 (作者：张华 发表于：2021-07-18)

1, virt-manager中创建一个名为cloud的network(192.168.100.0/24), 禁用dhcp

2, 创建一台1 CPU, 4G MEM, 20G raw DISK的KVM虚机(ubuntu/password)．
　　两块网卡，一块(enp1s0)使用默认的default network当管理用(其实不能，因为环境建好后这个网卡会失去ssh连接）, 一块(enp6s0)使用cloud network（从这块网络ssh进入即可，如果不能访问外网记得删除enp1s0上的默认路由）. 
   虚机启动后为cloud网卡配置下列IP:
cat <<EOF | sudo tee /etc/netplan/90-local.yaml
network:
  version: 2
  ethernets:
    enp6s0:
      dhcp4: no
      addresses:
      - 192.168.100.122/24
      gateway4: 192.168.15$i.1
      nameservers:
        addresses:
        - 192.168.100.1
EOF
sudo netplan apply
ubuntu@o7k:~$ cat /etc/netplan/01-netcfg.yaml 
network:
  version: 2
  renderer: networkd
  ethernets:
    enp1s0:
      dhcp4: yes

然后通过'ssh ubuntu@192.168.100.122'进入虚机，别用enp1s0因为它在完成部署后可以失去网络连接（被配置成了混杂模式）

3, 虚机的其他设置：
echo 'precedence ::ffff:0:0/96 100' |sudo tee -a /etc/gai.conf
echo 'ubuntu     ALL=(ALL) NOPASSWD:ALL' |sudo tee -a /etc/sudoers
cat <<EOF | sudo tee -a /etc/hosts
# and need to remove '127.0.1.1       o7k' as well
192.168.100.122 o7k
EOF

4, 特色网络配置
sudo cp /etc/apt/sources.list /etc/apt/sources.list_bak
sudo sed -i s/us.archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g /etc/apt/sources.list
sudo apt clean
sudo apt update
sudo apt upgrade -y
sudo apt install apt-transport-https ca-certificates curl gnupg2 software-properties-common -y
sudo apt install python3-dev libffi-dev gcc libssl-dev -y

sudo apt install python3-pip -y && sudo pip3 install --upgrade pip
#http://mirrors.cloud.tencent.com/pypi/simple
#http://pypi.douban.com/simple/
mkdir -p ~/.pip
cat <<EOF | sudo tee ~/.pip/pip.conf
[global]
index-url=https://pypi.tuna.tsinghua.edu.cn/simple
[install]
trusted-host=pypi.tuna.tsinghua.edu.cn
EOF
pip3 config list

5, 安装docker

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \\
   "deb [arch=amd64] https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
sudo apt-get update -y
sudo apt-get install docker-ce docker-ce-cli containerd.io -y
sudo systemctl enable docker && sudo systemctl start docker && sudo systemctl status docker
#https://registry.docker-cn.com
cat << EOF | sudo tee /etc/docker/daemon.json
{
"registry-mirrors":["https://3wk75oh9.mirror.aliyuncs.com"]                 
}
EOF
sudo systemctl daemon-reload && sudo systemctl restart docker
#sudo docker run -d --name pause busybox:latest /bin/sleep 10000000
sudo pip3 install docker

6, 安装并配置ansible
#git clone https://github.com/openstack/kolla-ansible -b stable/rocky
#pip install -r kolla-ansible/requirements.txt
sudo pip3 install -U 'ansible<2.9.19'
sudo pip3 install kolla-ansible==11.0.0  #victoria
sudo mkdir -p /etc/kolla
sudo chown $USER:$USER /etc/kolla
#copy globals.yml and passwords.yml to /etc/kolla
cp -r /usr/local/share/kolla-ansible/etc_examples/kolla/* /etc/kolla
#copy all-in-one and multinode to current dir
cp /usr/local/share/kolla-ansible/ansible/inventory/* .
cat <<EOF | tee /etc/kolla/globals.yml
kolla_base_distro: "ubuntu"
kolla_install_type: "source"
openstack_release: "victoria"
node_custom_config: "/etc/kolla/config"
kolla_internal_vip_address: "192.168.100.123"
#kolla_internal_fqdn: "kolla.quqi.com"
docker_namespace: "kolla"
docker_client_timeout: 120
network_interface: "enp6s0"
neutron_external_interface: "enp1s0"
neutron_plugin_agent: "openvswitch"
neutron_tenant_network_types: "vxlan,vlan,flat"
keepalived_virtual_router_id: "56"
enable_cinder: "yes"
enable_cinder_backup: "no"
enable_cinder_backend_lvm: "yes"
enable_heat: "no"
enable_neutron_provider_networks: "yes"
cinder_volume_group: "ceph-volumes"
nova_compute_virt_type: "kvm"
nova_console: "spice"
config_owner_user: "root"
config_owner_group: "root"
EOF
sudo mkdir -p /etc/ansible && sudo chown -R ubuntu /etc/ansible
cat << EOF > /etc/ansible/ansible.cfg
[defaults]
host_key_checking=False
pipelining=True
forks=100
EOF

#modify hostname
sudo sed -i s/localhost/o7k/g ./all-in-one
#change keystone password
kolla-genpwd
sed -i 's/^keystone_admin_password.*/keystone_admin_password: password/' /etc/kolla/passwords.yml

#create ceph vg
sudo mkdir -p /images && sudo chown $(whoami) /images
#fallocate -l 5G /var/lib/cinder_data.img
dd if=/dev/zero of=/images/ceph-volumes.img bs=1M count=5120 oflag=direct
#sudo losetup -d /dev/loop0 > /dev/null 2>&1
#sudo vgremove -y ceph-volumes > /dev/null 2>&1
#sudo vgcreate ceph-volumes $(sudo losetup --show -f /images/ceph-volumes.img)
#sudo lvcreate -L1G -nceph0 ceph-volumes
#sudo mkfs.xfs -f /dev/ceph-volumes/ceph0
cat <<EOF | sudo tee /etc/rc.local
#!/bin/sh -e
/usr/sbin/vgcreate ceph-volumes $(/usr/sbin/losetup --show -f /images/ceph-volumes.img)
exit 0
EOF
sudo chmod +x /etc/rc.local
sudo systemctl restart rc-local
sudo systemctl enable rc-local

cat >> /etc/kolla/config/ceph.conf << EOF
[global]
osd pool default size = 3
osd pool default min size = 2
mon_clock_drift_allowed = 2    
osd_pool_default_pg_num = 8
osd_pool_default_pgp_num = 8
mon clock drift warn backoff = 30
osd journal size = 100
EOF

mkdir -p /etc/kolla/config/nova
cat >> /etc/kolla/config/nova/nova-compute.conf << EOF
[libvirt]
inject_password=true
cpu_mode=host-passthrough
virt_type = kvm 
EOF
cat >> /etc/kolla/config/nova.conf << EOF
[DEFAULT]
service_down_time = 120
cpu_allocation_ratio = 4.0   
disk_allocation_ratio=1.0
ram_allocation_ratio = 1.0   
reserved_host_disk_mb = 1024 
reserved_host_memory_mb = 1024 
allow_resize_to_same_host = True    
remove_unused_base_images = False
image_cache_manager_interval = 0
resume_guests_state_on_host_boot = True
EOF
sudo mkdir -p /etc/kolla/config/horizon && sudo chown -R $(whoami) /etc/kolla
cat <<EOF | tee /etc/kolla/config/horizon/custom_local_settings
LAUNCH_INSTANCE_DEFAULTS = {'create_volume': False,}
EOF

7, 运行ansible
ansible -i all-in-one all -m ping
sudo pip3 install docker
kolla-ansible -i ./all-in-one prechecks -vv
#kolla-ansible -i ./all-in-one destroy --yes-i-really-really-mean-it && sudo netplan apply
kolla-ansible -i ./all-in-one bootstrap-servers -vv #install dependenty
kolla-ansible -i ./all-in-one deploy -vv

注意：此时外部网卡会失去连接（当然，我们是通过内部网卡ssh ubuntu@192.168.101.122连接的)
想上网的话：
sudo route del default gw 192.168.101.1
sudo route add default gw 192.168.100.1

8, 运行OpenStack CLI

sudo add-apt-repository cloud-archive:victoria
sudo apt update && apt dist-upgrade
sudo apt install python3-openstackclient -y

kolla-ansible -i all-in-one post-deploy
cp /etc/kolla/admin-openrc.sh ./
chmod +x admin-openrc.sh
source ~/admin-openrc.sh

# enp1s0=192.168.101.201 enp6s0=192.168.100.122
vim /usr/local/share/kolla-ansible/init-runonce
EXT_NET_CIDR=${EXT_NET_CIDR:-'192.168.101.0/24'}
EXT_NET_RANGE=${EXT_NET_RANGE:-'start=192.168.101.87,end=192.168.1.200'}
EXT_NET_GATEWAY=${EXT_NET_GATEWAY:-'192.168.101.1'}

. /usr/local/share/kolla-ansible/init-runonce
# visit http://192.168.100.123 admin password

#CLI
ssh-keygen -q -N ""
openstack keypair create --public-key ~/.ssh/id_rsa.pub mykey
openstack flavor create --ram 512 --disk 1 --vcpu 1 --id 122 myflavor
cat << EOF > user-data
#cloud-config
user: ubuntu
password: password
chpasswd: { expire: False }
EOF
openstack server create --wait --image cirros --flavor myflavor --key-name mykey --network=demo-net --user-data ./user-data --config-drive true i1

# Don't know why there is no subnet for the network public1, so create it
openstack subnet create --subnet-range 192.168.101.0/24 --network public1 --allocation-pool start=192.168.101.87,end=192.168.101.200 --gateway 192.168.101.1 public1-subnet
#openstack router unset demo-router
openstack router set --external-gateway public1 demo-router
#openstack router remove subnet demo-router demo-subnet
openstack router add subnet demo-router demo-subnet

fix_ip=$(openstack server list -f value |awk '/i1/ {print $4}' |awk -F '=' '{print $2}' |awk -F ',' '{print $1}')
ext_net=$(openstack network show public1 -f value -c id)
fip=$(openstack floating ip create $ext_net -f value -c floating_ip_address)
openstack floating ip set $fip --fixed-ip-address $fix_ip --port $(openstack port list --fixed-ip ip-address=$fix_ip -c id -f value)

SG=$(openstack security group list --project $(openstack project show admin -fvalue -cid) |awk '/default/ {print $2}')
openstack security group rule create $SG --protocol icmp --remote-ip 0.0.0.0/0
openstack security group rule create $SG --protocol tcp --dst-port 22

openstack console log show i1
#login as 'cirros' user. default password: 'gocubsgo'. use 'sudo' for root.

ubuntu@o7k:~$ nova list
+--------------------------------------+------+--------+------------+-------------+--------------------------------------+
| ID                                   | Name | Status | Task State | Power State | Networks                             |
+--------------------------------------+------+--------+------------+-------------+--------------------------------------+
| 9945ba16-bbb5-4a30-b0fb-0e2e59a85a98 | i1   | ACTIVE | -          | Running     | demo-net=10.0.0.154, 192.168.101.181 |
+--------------------------------------+------+--------+------------+-------------+--------------------------------------+

8, 结果，只是在o7k虚机上无法访问里面虚机i1的FIP，但这个FIP可以在其他机器访问的．正确
sudo docker ps |grep neutron
sudo docker exec -ti -uroot neutron_l3_agent bash
(neutron-l3-agent)[root@o7k /]# ip netns exec qrouter-38ba43f7-d3d7-4770-890d-06a9b1845e89 ping 10.0.0.154
PING 10.0.0.154 (10.0.0.154) 56(84) bytes of data.
64 bytes from 10.0.0.154: icmp_seq=1 ttl=64 time=0.512 ms
(neutron-l3-agent)[root@o7k /]# ip netns exec qrouter-38ba43f7-d3d7-4770-890d-06a9b1845e89 ping 192.168.101.1
PING 192.168.101.1 (192.168.101.1) 56(84) bytes of data.
64 bytes from 192.168.101.1: icmp_seq=1 ttl=64 time=0.582 ms

hua@node1:~$ ping 192.168.101.181 -c1
PING 192.168.101.181 (192.168.101.181) 56(84) bytes of data.
64 bytes from 192.168.101.181: icmp_seq=1 ttl=63 time=1.13 ms

9, 如何调试

#for all projects: kolla_dev_mode: true
cat <<EOF | sudo tee -a /etc/kolla/globals.yml
neutron_dev_mode: true
EOF
kolla-ansible -i all-in-one stop --yes-i-really-really-mean-it
kolla-ansible -i all-in-one deploy -vvv
cd /opt/stack/neutron  #modify your code (eg: neutron/agent/l3_agent.py) to add 'import rpdb;rpdb.set_trace()'
sudo docker exec -ti -uroot neutron_l3_agent pip install rpdb
sudo docker restart neutron_l3_agent
nc 127.0.0.1 4444

ubuntu@o7k:/opt/stack/neutron$ sudo netstat -anp |grep 4444
tcp        0      0 127.0.0.1:4444          0.0.0.0:*               LISTEN      114670/neutron-l3-a 
ubuntu@o7k:/opt/stack/neutron$ nc 127.0.0.1 4444
> /var/lib/kolla/venv/lib/python3.8/site-packages/neutron/agent/l3_agent.py(49)main()
-> register_opts(cfg.CONF)
(Pdb)