docker + calico网络，实现不同容器之间的相互访问

Posted 2020-12-31 alber

docker use calico
#基础环境
IP              主机名  系统版本    安装组件
192.168.56.151  node1   centos7.4   docker、calicoctl、etcd
192.168.56.152  node2   centos7.4   docker、calicoctl、etcd
192.168.56.153  node3   centos7.4   docker、calicoctl、etcd


###docker
#所有节点执行安装docker
yum install docker -y
systemctl start docker
systemctl enable docker




###etcd
#所有节点执行安装etcd
yum install etcd -y

#NODE-1
cat > /etc/etcd/etcd.conf <<EOF
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"

ETCD_NAME="node1"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.56.151:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.56.151:2379"
ETCD_INITIAL_CLUSTER="node1=http://192.168.56.151:2380,node2=http://192.168.56.152:2380,node3=http://192.168.56.153:2380"
EOF

#NODE-2
cat > /etc/etcd/etcd.conf <<EOF
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"

ETCD_NAME="node2"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.56.152:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.56.152:2379"
ETCD_INITIAL_CLUSTER="node1=http://192.168.56.151:2380,node2=http://192.168.56.152:2380,node3=http://192.168.56.153:2380"
EOF

#NODE-3
cat > /etc/etcd/etcd.conf <<EOF
#[Member]
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"

ETCD_NAME="node3"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.56.153:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.56.153:2379"
ETCD_INITIAL_CLUSTER="node1=http://192.168.56.151:2380,node2=http://192.168.56.152:2380,node3=http://192.168.56.153:2380"
EOF

systemctl start etcd
systemctl enable etcd

etcdctl member list

##修改docker支持etcd

#node-1
#ExecStart后增加
--cluster-store=etcd://192.168.56.151:2379

#node-2
#ExecStart后增加
--cluster-store=etcd://192.168.56.152:2379

#node-3
#ExecStart后增加
--cluster-store=etcd://192.168.56.153:2379

#####CALICO
#node-1
docker run --net=host --privileged --name=calico-node -d --restart=always -e NODENAME=node1 -e CALICO_NETWORKING_BACKEND=bird -e CALICO_LIBNETWORK_ENABLED=true -e IP=192.168.56.151 -e ETCD_ENDPOINTS=http://127.0.0.1:2379 
-v /var/log/calico:/var/log/calico -v /var/run/calico:/var/run/calico -v /lib/modules:/lib/modules -v /run:/run -v /run/docker/plugins:/run/docker/plugins -v /var/run/docker.sock:/var/run/docker.sock quay.io/calico/node:v2.6.10


#node-2
docker run --net=host --privileged --name=calico-node -d --restart=always -e NODENAME=node2 -e CALICO_NETWORKING_BACKEND=bird -e CALICO_LIBNETWORK_ENABLED=true -e IP=192.168.56.152 -e ETCD_ENDPOINTS=http://127.0.0.1:2379 
-v /var/log/calico:/var/log/calico -v /var/run/calico:/var/run/calico -v /lib/modules:/lib/modules -v /run:/run -v /run/docker/plugins:/run/docker/plugins -v /var/run/docker.sock:/var/run/docker.sock quay.io/calico/node:v2.6.10


#node-3
docker run --net=host --privileged --name=calico-node -d --restart=always -e NODENAME=node3 -e CALICO_NETWORKING_BACKEND=bird -e CALICO_LIBNETWORK_ENABLED=true -e IP=192.168.56.153 -e ETCD_ENDPOINTS=http://127.0.0.1:2379 
-v /var/log/calico:/var/log/calico -v /var/run/calico:/var/run/calico -v /lib/modules:/lib/modules -v /run:/run -v /run/docker/plugins:/run/docker/plugins -v /var/run/docker.sock:/var/run/docker.sock quay.io/calico/node:v2.6.10


#查看calico状态
[[email protected] ~]# calicoctl node status
Calico process is running.

IPv4 BGP status
+----------------+-------------------+-------+----------+-------------+
|  PEER ADDRESS  |     PEER TYPE     | STATE |  SINCE   |    INFO     |
+----------------+-------------------+-------+----------+-------------+
| 192.168.56.152 | node-to-node mesh | up    | 14:29:26 | Established |
| 192.168.56.153 | node-to-node mesh | up    | 14:31:16 | Established |
+----------------+-------------------+-------+----------+-------------+

IPv6 BGP status
No IPv6 peers found.


###测试

calicoctl get ipPool

cat >ipPool <<EOF
- apiVersion: v1
  kind: ipPool
  metadata:
    cidr: 10.20.0.0/24
  spec:
    ipip:
      enabled: true
    nat-outgoing: true
EOF

calicoctl create -f ipPool.yaml

####连通性验证
在上面创建的ip pool（10.20.0.0/24）里创建子网络，如：

docker network create --driver calico --ipam-driver calico-ipam  --subnet 10.20.0.0/24 net1
docker network create --driver calico --ipam-driver calico-ipam  --subnet 10.20.0.0/24 net2
docker network create --driver calico --ipam-driver calico-ipam  --subnet 10.20.0.0/24 net3

在node1和node2上分别创建几个容器来测试下容器网络的连通性。
#node1
docker run --net net1 --name workload-A -tid busybox
docker run --net net2 --name workload-B -tid busybox
docker run --net net1 --name workload-C -tid busybox
#node2
docker run --net net3 --name workload-D -tid busybox
docker run --net net1 --name workload-E -tid busybox

可以在node1上使用如下命令来试验连通性：

#同一网络内的容器（即使不在同一节点主机上）可以使用容器名来访问
docker exec workload-A ping -c 4 workload-C.net1
docker exec workload-A ping -c 4 workload-E.net1
#不同网络内的容器需要使用容器ip来访问（使用容器名会报：bad address）
docker exec workload-A ping -c 2  `docker inspect --format "{{ .NetworkSettings.Networks.net2.IPAddress }}" workload-B`