配置Etcd集群和TLS认证

Posted 2020-10-30

由于后续准备在内网开发和测试环境采用二进制方式部署K8S相关组件，并考虑各组件的高可用性和安全性问题,本节介绍etcd服务的集群及tls配置。

一、安装环境介绍

二、Etcd二进制软件包下载地址:
https://github.com/coreos/etcd/releases/download/v3.3.2/etcd-v3.3.2-linux-amd64.tar.gz

三、安装与配置etcd组件
1、删除rpm版本的软件包、设置各自的主机名及时间

# yum -y remove etcd
# hostnamectl  set-hostname vm1
# timedatectl set-timezone Asia/Shanghai
# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.115.5 vm1
192.168.115.6 vm2
192.168.115.7 vm3
# ntpdate -u pool.ntp.org

2、关闭防火墙、配置秘钥信任

# systemctl stop firewalled 
# systemctl disable firewalled 
# ssh-keygen 
# ssh-copy-id -i /root/.ssh/id_rsa.pub  [email protected]
# ssh-copy-id -i /root/.ssh/id_rsa.pub  [email protected]
# date && ssh vm2  date  && ssh vm3 date      

3、将etcd软件包上传并解压到/usr/local/bin目录

# cd /usr/local/src/
# tar -zxvpf etcd-v3.3.2-linux-amd64.tar.gz 
# cp etcd-v3.3.2-linux-amd64/{etcd,etcdctl} /usr/local/sbin/
# chmod +x /usr/local/sbin/etcd*
# scp -rp /usr/local/sbin/etcd* vm2:/usr/local/sbin/
# scp -rp /usr/local/sbin/etcd* vm3:/usr/local/sbin/

4、准备配置文件
Vm1:

# cat /etc/etcd.conf
name: infra0
data-dir: /data/etcd
listen-client-urls: http://192.168.115.5:2379,http://127.0.0.1:2379
advertise-client-urls: http://192.168.115.5:2379,http://127.0.0.1:2379
listen-peer-urls: http://192.168.115.5:2380
initial-advertise-peer-urls: http://192.168.115.5:2380
initial-cluster: infra0=http://192.168.115.5:2380,infra1=http://192.168.115.6:2380,infra2=http://192.168.115.7:2380
initial-cluster-token: etcd-cluster-token
initial-cluster-state: new

Vm2:

# cat /etc/etcd.conf
name: infra1  
data-dir: /data/etcd
listen-client-urls: http://192.168.115.6:2379,http://127.0.0.1:2379  
advertise-client-urls: http://192.168.115.6:2379,http://127.0.0.1:2379  
listen-peer-urls: http://192.168.115.6:2380  
initial-advertise-peer-urls: http://192.168.115.6:2380  
initial-cluster: infra0=http://192.168.115.5:2380,infra1=http://192.168.115.6:2380,infra2=http://192.168.115.7:2380
initial-cluster-token: etcd-cluster-token
initial-cluster-state: new

VM3:

# cat /etc/etcd.conf
name: infra2  
data-dir: /data/etcd
listen-client-urls: http://192.168.115.7:2379,http://127.0.0.1:2379  
advertise-client-urls: http://192.168.115.7:2379,http://127.0.0.1:2379  
listen-peer-urls: http://192.168.115.7:2380  
initial-advertise-peer-urls: http://192.168.115.7:2380  
initial-cluster: infra0=http://192.168.115.5:2380,infra1=http://192.168.115.6:2380,infra2=http://192.168.115.7:2380
initial-cluster-token: etcd-cluster-token
initial-cluster-state: new

5、启动etcd集群并测试

# mkdir -p /data/etcd
# nohup etcd --config-file=/etc/etcd.conf &
# export ETCDCTL_API=2
# etcdctl cluster-health
# etcdctl member list 
# export ETCDCTL_API=3
# etcdctl --write-out=table --endpoints=192.168.115.5:2379 member list

四、配置etcd tls
1、下载cfssl工具

# mkdir ~/bin
# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
# mv cfssl_linux-amd64 /usr/local/bin/cfssl
# mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfss-certinfo
# chmod +x /usr/local/bin/cfssl* 

2、生成证书

# mkdir ssl
# cd ssl
# cat bulid-key.sh 
echo ‘{"CN":"CA","key":{"algo":"rsa","size":2048}}‘ | cfssl gencert -initca - | cfssljson -bare ca -
echo ‘{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","server auth","client auth"]}}}‘ > ca-config.json
export ADDRESS=192.168.115.5,192.168.115.6,192.168.115.7,vm1,vm2,vm3
export NAME=server
echo ‘{"CN":"‘$NAME‘","hosts":[""],"key":{"algo":"rsa","size":2048}}‘ | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
export ADDRESS=
export NAME=client
echo ‘{"CN":"‘$NAME‘","hosts":[""],"key":{"algo":"rsa","size":2048}}‘ | cfssl gencert -config=ca-config.json -ca=ca.pem -ca-key=ca-key.pem -hostname="$ADDRESS" - | cfssljson -bare $NAME
# sh bulid-key.sh

# ll
total 44
-rw-r--r-- 1 root root  732 Apr  3 05:13 build-ca.sh
-rw-r--r-- 1 root root  112 Apr  3 05:13 ca-config.json
-rw-r--r-- 1 root root  883 Apr  3 05:13 ca.csr
-rw------- 1 root root 1675 Apr  3 05:13 ca-key.pem
-rw-r--r-- 1 root root 1119 Apr  3 05:13 ca.pem
-rw-r--r-- 1 root root  928 Apr  3 05:13 client.csr
-rw------- 1 root root 1675 Apr  3 05:13 client-key.pem
-rw-r--r-- 1 root root 1180 Apr  3 05:13 client.pem
-rw-r--r-- 1 root root  928 Apr  3 05:13 server.csr
-rw------- 1 root root 1679 Apr  3 05:13 server-key.pem
-rw-r--r-- 1 root root 1220 Apr  3 05:13 server.pem

4、将相关的文件复制到etc节点上

# mkdir -p /etc/ssl/etcd/ 
# cp ./*.pem  /etc/ssl/etcd/
# scp -rp /etc/ssl/etcd/ vm2:/etc/ssl/
# scp -rp /etc/ssl/etcd/ vm3:/etc/ssl/

5、配置etcd启动加载相关证书
Vm1:

# etcd --name=infra0 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.5:2380 --initial-advertise-peer-urls=https://192.168.115.5:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=true

vm2：

# etcd --name=infra1 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.6:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.6:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.6:2380 --initial-advertise-peer-urls=https://192.168.115.6:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=true

Vm3:

# etcd --name=infra2 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.7:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.7:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.7:2380 --initial-advertise-peer-urls=https://192.168.115.7:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=true

6、验证

# export ETCDCTL_API=2
# etcdctl --cert-file=/etc/ssl/etcd/client.pem   --key-file=/etc/ssl/etcd/client-key.pem  --ca-file=/etc/ssl/etcd/ca.pem --endpoints=https://192.168.115.5:2379,https://192.168.115.6:2379,https://192.168.115.7:2379 cluster-health 
# export ETCDCTL_API=3
# etcdctl --write-out=table --cert=/etc/ssl/etcd/client.pem --key=/etc/ssl/etcd/client-key.pem --cacert=/etc/ssl/etcd/ca.pem --endpoints=https://192.168.115.5:2379,https://192.168.115.6:2379,https://192.168.115.7:2379
member list

6、配置自启动脚本

# cat /usr/lib/systemd/system/etcd.service    
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
WorkingDirectory=/data/etcd/
EnvironmentFile=-/etc/etcd.conf
User=root
# set GOMAXPROCS to number of processors
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/local/sbin/etcd --name=infra0 --data-dir=/data/etcd --listen-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.115.5:2379,https://127.0.0.1:2379 --listen-peer-urls=https://192.168.115.5:2380 --initial-advertise-peer-urls=https://192.168.115.5:2380 --initial-cluster=infra0=https://192.168.115.5:2380,infra1=https://192.168.115.6:2380,infra2=https://192.168.115.7:2380 --initial-cluster-token=etcd-cluster-token --initial-cluster-state=new --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --peer-cert-file=/etc/ssl/etcd/server.pem --peer-key-file=/etc/ssl/etcd/server-key.pem --trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-trusted-ca-file=/etc/ssl/etcd/ca.pem --peer-client-cert-auth=true --client-cert-auth=true"
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

# systemctl daemon-reload 

参考：
https://coreos.com/os/docs/latest/generate-self-signed-certificates.html