Kubernetes二进制部署——证书的制作和ETCD的部署

Posted 2022-02-02 IHBOS

Kubernetes二进制部署——证书的制作和ETCD的部署

• 一、实验环境
• • 自签 SSL 证书
• 二、ETCD集群部署
• • 1、环境部署
  • 2、master节点
  • 3、node1节点
  • 4、node2节点
  • 5、master节点

一、实验环境

主机名	IP	服务
master	192.168.172.10/24	kube-apiserver kube-controller-manager kube-scheduler etcd
node1	192.168.172.20/24	etcd docker kubelet kube-proxyflannel
node2	192.168.172.30/24	etcd docker kubelet kube-proxyflannel

自签 SSL 证书

组件	使用的证书
etcd	ca.pem、server.pem、server-key.pem
fiannel	ca.pem、server.pem、server-key.pem
kube-apiserver	ca.pem、server.pem、server-key.pem
kubelet	ca.pem、ca-key.pem
kube-proxy	ca.pem、kube-proxy.pem、kube-proxy-key.pem
kubectl	ca.pem、admin.pem、admin-key.pem

二、ETCD集群部署

二进制部署方式

1、环境部署

更改matser主机名'
hostnamectl set-hostname master
su -


另外两台node1、2节点'
hostnamectl set-hostname node1
 su -
hostnamectl set-hostname node2
 su -

关闭防火墙及安全访问控制机制'
systemctl stop firewalld
systemctl disable firewalld.service 
setenforce 0

2、master节点

准备工作

[root@master ~]# mkdir k8s
[root@master ~]# cd k8s/
[root@master ~/k8s]# vim cfssl.sh          #此脚本用于下载证书制作工具
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

[root@master ~/k8s]# bash cfssl.sh           #执行该脚本

开始制作证书

[root@master ~/k8s]# mkdir etcd-cert
[root@master ~/k8s]# cd etcd-cert/
##定义ca证书
cat > ca-config.json <<EOF

  "signing": 
    "default": 
      "expiry": "87600h"
    ,
    "profiles": 
      "www": 
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"     
        ]
      
    
  

EOF

##实现证书签名
cat > ca-csr.json <<EOF 
   
    "CN": "etcd CA",
    "key": 
        "algo": "rsa",
        "size": 2048
    ,
    "names": [
        
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        
    ]

EOF

##生产证书，生成ca-key.pem  ca.pem
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

##指定etcd三个节点之间的通信验证
cat > server-csr.json <<EOF

    "CN": "etcd",
    "hosts": [
    "192.168.172.10",
    "192.168.172.20",
    "192.168.172.30"
    ],
    "key": 
        "algo": "rsa",
        "size": 2048
    ,
    "names": [
        
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        
    ]

EOF

##生成ETCD证书 server-key.pem   server.pem
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

[root@master ~/k8s/etcd-cert]# cd ..
#拉入压缩包etcd-v3.3.10-linux-amd64.tar.gz 
[root@master ~/k8s]# tar zxvf etcd-v3.3.10-linux-amd64.tar.gz 
[root@master ~/k8s]# ls etcd-v3.3.10-linux-amd64

[root@master ~/k8s]# mkdir /opt/etcd/cfg,bin,ssl -p   #三个目录分别存放配置文件，命令文件，证书
[root@master ~/k8s]# mv etcd-v3.3.10-linux-amd64/etcd etcd-v3.3.10-linux-amd64/etcdctl /opt/etcd/bin/
[root@master ~/k8s]# ls /opt/etcd/bin/
etcd  etcdctl
[root@master ~/k8s]# cp etcd-cert/*.pem /opt/etcd/ssl/
[root@master ~/k8s]# ls /opt/etcd/ssl/
ca-key.pem  ca.pem  server-key.pem  server.pem

[root@master ~/k8s]# vim etcd.sh
#!/bin/bash
#以下为使用格式：etcd名称 当前etcd的IP地址+完整的集群名称和地址
# example: ./etcd.sh etcd01 192.168.172.10 etcd02=https://192.168.172.20:2380,etcd03=https://192.168.172.30:2380

ETCD_NAME=$1						#位置变量1：etcd节点名称
ETCD_IP=$2							#位置变量2：节点地址
ETCD_CLUSTER=$3						#位置变量3：集群

WORK_DIR=/opt/etcd					#指定工作目录

cat <<EOF >$WORK_DIR/cfg/etcd				#在指定工作目录创建ETCD的配置文件
#[Member]
ETCD_NAME="$ETCD_NAME"				#etcd名称
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://$ETCD_IP:2380"		#etcd IP地址：2380端口。用于集群之间通讯
ETCD_LISTEN_CLIENT_URLS="https://$ETCD_IP:2379"	#etcd IP地址：2379端口，用于开放给外部客户端通讯

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://$ETCD_IP:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://$ETCD_IP:2379"	#对外提供的url使用https的协议进行访问
ETCD_INITIAL_CLUSTER="etcd01=https://$ETCD_IP:2380,$ETCD_CLUSTER"		#多路访问
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"		#tokens 令牌环名称：etcd-cluster
ETCD_INITIAL_CLUSTER_STATE="new"			#状态，重新创建
EOF

cat <<EOF >/usr/lib/systemd/system/etcd.service		#定义ectd的启动脚本
[Unit]										#基本项			
Description=Etcd Server						#类似为 etcd 服务
After=network.target						
After=network-online.target
Wants=network-online.target

[Service]									#服务项
Type=notify
EnvironmentFile=$WORK_DIR/cfg/etcd		#etcd文件位置
ExecStart=$WORK_DIR/bin/etcd \\			#准启动状态及以下的参数
--name=\\$ETCD_NAME \\
--data-dir=\\$ETCD_DATA_DIR \\
--listen-peer-urls=\\$ETCD_LISTEN_PEER_URLS \\
--listen-client-urls=\\$ETCD_LISTEN_CLIENT_URLS,http://127.0.0.1:2379 \\
--advertise-client-urls=\\$ETCD_ADVERTISE_CLIENT_URLS \\ #以下为群集内部的设定
--initial-advertise-peer-urls=\\$ETCD_INITIAL_ADVERTISE_PEER_URLS \\
--initial-cluster=\\$ETCD_INITIAL_CLUSTER \\
--initial-cluster-token=\\$ETCD_INITIAL_CLUSTER_TOKEN \\	#群集内部通信，也是使用的令牌，为了保证安全（防范中间人窃取）
--initial-cluster-state=new \\
--cert-file=$WORK_DIR/ssl/server.pem \\		#证书相关参数
--key-file=$WORK_DIR/ssl/server-key.pem \\
--peer-cert-file=$WORK_DIR/ssl/server.pem \\
--peer-key-file=$WORK_DIR/ssl/server-key.pem \\
--trusted-ca-file=$WORK_DIR/ssl/ca.pem \\
--peer-trusted-ca-file=$WORK_DIR/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536								#开放最多的端口号

[Install]
WantedBy=multi-user.target						#进行启动
EOF

systemctl daemon-reload							#参数重载
systemctl enable etcd
systemctl restart etcd

进入卡住状态等待其他节点加入
打开一个新的终端，会发现etcd进程已经开启

拷贝证书去其他节点

[root@master ~]# scp -r /opt/etcd/ root@192.168.172.20:/opt/
[root@master ~]# scp -r /opt/etcd/ root@192.168.172.30:/opt/

启动脚本拷贝其他节点

[root@master ~]# scp /usr/lib/systemd/system/etcd.service  root@192.168.172.20:/usr/lib/systemd/system/
[root@master ~]# scp /usr/lib/systemd/system/etcd.service  root@192.168.172.30:/usr/lib/systemd/system/

3、node1节点

[root@node1 ~]# vim /opt/etcd/cfg/etcd

#[Member]
ETCD_NAME="etcd02"         #修改数据库名称
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.172.20:2380"           #修改节点IP
ETCD_LISTEN_CLIENT_URLS="https://192.168.172.20:2379"         #修改节点IP

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.172.20:2380"  #修改节点IP
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.172.20:2379"        #修改节点IP
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.172.10:2380,etcd02=https://192.168.172.20:2380,etcd03=https://192.168.172.30:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

[root@node1 ~]# systemctl stop firewalld.service 
[root@node1 ~]# setenforce 0
[root@node1 ~]# systemctl start etcd.service

4、node2节点

[root@node2 ~]# vim /opt/etcd/cfg/etcd

#[Member]
ETCD_NAME="etcd03"         #修改数据库名称
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.172.30:2380"           #修改节点IP
ETCD_LISTEN_CLIENT_URLS="https://192.168.172.30:2379"         #修改节点IP

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.172.30:2380"  #修改节点IP
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.172.30:2379"        #修改节点IP
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.172.10:2380,etcd02=https://192.168.172.20:2380,etcd03=https://192.168.172.30:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

[root@node2 ~]# systemctl stop firewalld.service 
[root@node2 ~]# setenforce 0
[root@node2 ~]# systemctl start etcd.service

5、master节点

[root@master ~]# systemctl start etcd
[root@master ~]# systemctl enable etcd
[root@master ~]# systemctl status etcd
检查群集状态(需要在有证书的目录下使用此命令)
[root@master ~]# cd /opt/etcd/ssl
[root@master /opt/etcd/ssl]# /opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.172.10:2379,https://192.168.172.20:2379,https://192.168.172.30:2379" cluster-health