开发环境redis-6.2.4 cluster搭建

Posted 2022-12-03 y_zilong

由于Redis Labs Redis 6.0.3之前版本存在拒绝服务漏洞。该漏洞源于lua_struct.c中的“getnum”函数中的整数溢出。远程攻击者可利用该漏洞通过发送大量的特制命令导致堆栈缓冲区溢出，从而造成拒绝服务

现在进行redis cluster集群升级

10.8.17.227 端口6389 6390

10.8.17.228 端口6389 6390

10.8.17.229 端口6389 6390

0、关闭老集群

#按照每台主机关闭
[root@BJ-HW-VM-17-227 ~]# cd /apps/usr/aspire/redis-5.0.8/
[root@BJ-HW-VM-17-227 redis-5.0.8]# ls
00-RELEASENOTES  COPYING  logs       README.md   runtest-cluster    sentinel.conf  utils
BUGS             deps     Makefile   redis.conf  runtest-moduleapi  src
CONTRIBUTING     INSTALL  MANIFESTO  runtest     runtest-sentinel   tests
[root@BJ-HW-VM-17-227 redis-5.0.8]# cd src/
[root@BJ-HW-VM-17-227 src]# ./redis-cli -h 10.8.17.227 -p 6379 shutdown
[root@BJ-HW-VM-17-227 src]# ./redis-cli -h 10.8.17.227 -p 6380 shutdown

[root@BJ-HW-VM-17-228 ~]# cd /apps/usr/aspire/redis-5.0.8/src/
[root@BJ-HW-VM-17-228 src]# ./redis-cli -h 10.8.17.228 -p 6379 shutdown
[root@BJ-HW-VM-17-228 src]# ./redis-cli -h 10.8.17.228 -p 6380 shutdown

[root@BJ-HW-VM-17-229 ~]# cd /apps/usr/aspire/redis-5.0.8/src/
[root@BJ-HW-VM-17-229 src]# ./redis-cli -h 10.8.17.229 -p 6379 shutdown
[root@BJ-HW-VM-17-229 src]# ./redis-cli -h 10.8.17.229 -p 6380 shutdown

1、三台主机上面安装redis-6.2.4，跑脚本

vim install_redis-6.2.4.sh

#!/bin/bash
#
#********************************************************************
#Author:		    yzl
#QQ: 			    [e-mail]
#Date: 			    2022-01-04
#FileName：		    install_redis-6.2.4.sh
#URL: 			    yzil.cn
#Description：		The test script
#Copyright (C): 	2022 All rights reserved
#********************************************************************

VERSION=redis-6.2.4
PASSWORD=123456
INSTALL_DIR=/apps/usr/redis-6.2.4

color () 
    RES_COL=60
    MOVE_TO_COL="echo -en \\\\033[$RES_COLG"
    SETCOLOR_SUCCESS="echo -en \\\\033[1;32m"
    SETCOLOR_FAILURE="echo -en \\\\033[1;31m"
    SETCOLOR_WARNING="echo -en \\\\033[1;33m"
    SETCOLOR_NORMAL="echo -en \\E[0m"
    echo -n "$1" && $MOVE_TO_COL
    echo -n "["
    if [ $2 = "success" -o $2 = "0" ] ;then
        $SETCOLOR_SUCCESS
        echo -n $"  OK  "    
    elif [ $2 = "failure" -o $2 = "1"  ] ;then 
        $SETCOLOR_FAILURE
        echo -n $"FAILED"
    else
        $SETCOLOR_WARNING
        echo -n $"WARNING"
    fi
    $SETCOLOR_NORMAL
    echo -n "]"
    echo 



install() 
yum  -y install gcc jemalloc-devel

 #   if [  -e $VERSION$TAR ];then
        color "相关文件已准备好" 0
 #   else
 #       color '开始下载 redis 源码包' 0
 #       wget http://download.redis.io/releases/$VERSION.tar.gz ||  color "Redis 源码下载失败" 1 ; exit; 
 #       [ $? -ne 0 ] &&  color "下载 $redis_FILE$TAR文件失败" 1; exit;  
 #   fi


tar xf $VERSION.tar.gz
cd $VERSION
make -j 4 PREFIX=$INSTALL_DIR install && color "Redis 编译安装完成" 0 ||  color "Redis 编译安装失败" 1 ;exit ; 


ln -s $INSTALL_DIR/bin/redis-*  /usr/bin/
mkdir -p $INSTALL_DIR/etc,log,data,run
cp redis.conf  $INSTALL_DIR/etc/

sed -i -e 's/bind 127.0.0.1/bind 0.0.0.0/' -e "s/appendfilename "appendonly.aof"/appendfilename "appendonly-6389.aof"/" -e "s/dbfilename dump.rdb/dbfilename dump_6389.rdb/"  -e "s/port 6379/port 6389/" -e "/# requirepass/a requirepass $PASSWORD"  -e "/^dir .*/c dir $INSTALL_DIR/data/"  -e "/logfile .*/c logfile $INSTALL_DIR/log/redis-6389.log"  -e  "/^pidfile .*/c  pidfile $INSTALL_DIR/run/redis_6389.pid" $INSTALL_DIR/etc/redis.conf

sleep 3
mv $INSTALL_DIR/etc/redis.conf $INSTALL_DIR/etc/redis-6389.conf

#if id redis &> /dev/null ;then 
#    color "Redis 用户已存在" 1 
#else
#    useradd -r -s /sbin/nologin redis
#    color "Redis 用户创建成功" 0
#fi

if id aspire &> /dev/null ;then 
    color "aspire 用户已存在" 1 
else
    useradd -r -s /sbin/nologin aspire
    color "aspire 用户创建成功" 0
fi

#chown -R redis.redis $INSTALL_DIR
chown -R aspire.aspire $INSTALL_DIR

#cat >> /etc/sysctl.conf <<EOF
#net.core.somaxconn = 2048
#vm.overcommit_memory = 1
#EOF
#sysctl -p 

#echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.d/rc.local
#chmod +x /etc/rc.d/rc.local
#/etc/rc.d/rc.local

cat > /usr/lib/systemd/system/redis-6389.service <<EOF
[Unit]
Description=Redis persistent key-value database
After=network.target

[Service]
ExecStart=$INSTALL_DIR/bin/redis-server $INSTALL_DIR/etc/redis-6389.conf --supervised systemd
ExecStop=/bin/kill -s QUIT \\$MAINPID
#Type=notify
User=aspire
Group=aspire
RuntimeDirectory=/apps/usr
RuntimeDirectoryMode=0755

[Install]
WantedBy=multi-user.target

EOF
systemctl daemon-reload 
systemctl enable --now  redis-6389 &> /dev/null && color "Redis 服务启动成功,Redis信息如下:"  0 ||  color "Redis 启动失败" 1 ;exit;  
sleep 2
redis-cli -a $PASSWORD INFO Server 2> /dev/null



install 

2、每台主机复制一个多实例

#分别复制redis-6389.conf 

cp -p /apps/usr/redis-6.2.4/etc/redis-6389.conf /apps/usr/redis-6.2.4/etc/redis-6390.conf
sed -i 's/6389/6390/' /apps/usr/redis-6.2.4/etc/redis-6390.conf

[root@BJ-HW-VM-17-227 ~]# cp -p /apps/usr/redis-6.2.4/etc/redis-6389.conf /apps/usr/redis-6.2.4/etc/redis-6390.conf
[root@BJ-HW-VM-17-227 ~]# cat /apps/usr/redis-6.2.4/etc/redis-6390.conf |grep 6389
port 6389
# tls-port 6389
pidfile /apps/usr/redis-6.2.4/run/redis_6389.pid
logfile /apps/usr/redis-6.2.4/log/redis-6389.log
dbfilename dump_6389.rdb
# cluster-announce-tls-port 6389
[root@BJ-HW-VM-17-227 ~]# sed -i 's/6389/6390/' /apps/usr/redis-6.2.4/etc/redis-6390.conf
[root@BJ-HW-VM-17-227 ~]# cat /apps/usr/redis-6.2.4/etc/redis-6390.conf |grep 6389
[root@BJ-HW-VM-17-227 ~]# cat /apps/usr/redis-6.2.4/etc/redis-6390.conf |grep 6390
port 6390
# tls-port 6390
pidfile /apps/usr/redis-6.2.4/run/redis_6390.pid
logfile /apps/usr/redis-6.2.4/log/redis-6390.log
dbfilename dump_6390.rdb
# cluster-announce-tls-port 6390

#分别复制redis-6389.service 
cp -p /usr/lib/systemd/system/redis-6389.service /usr/lib/systemd/system/redis-6390.service

sed -i 's/6389/6390/' /usr/lib/systemd/system/redis-6390.service

systemctl daemon-reload

#后面检测到appendonly.aof没改成功
#分别改appendonly.aof
cat /apps/usr/redis-6.2.4/etc/redis-6389.conf |grep appendonly.aof
cat /apps/usr/redis-6.2.4/etc/redis-6390.conf |grep appendonly.aof

sed -i 's/appendfilename "appendonly.aof"/appendfilename "appendonly-6389.aof"/' /apps/usr/redis-6.2.4/etc/redis-6389.conf

sed -i 's/appendfilename "appendonly.aof"/appendfilename "appendonly-6390.aof"/' /apps/usr/redis-6.2.4/etc/redis-6390.conf

3、开启rdb

#先开启rbd、aof集群启动在开启
#六台主机都加上以下
save 900 1
save 300 10
save 60 10000

4、启动cluster功能

##手动修改配置文件
[root@cent7_6 ~]# vim /apps/redis/etc/redis.conf 
bind 0.0.0.0
requirepass 123456
masterauth 123456    #后期的master和slave主从复制
cluster-enabled yes  #开启集群，开启后redis进程会有cluster标识
cluster-config-file nodes-6379.conf   #集群状态文件，记录主从关系及slot范围信息，由redis cluster集群自动创建和维护
cluster-require-full-coverage no   #设为no可以防止一个节点不可用导致整个cluster不可用

#查看

[root@BJ-HW-VM-17-227 ~]# grep '^bind' /apps/usr/redis-6.2.4/etc/redis-6389.conf 
bind 0.0.0.0 -::1
[root@BJ-HW-VM-17-227 ~]# grep '^requirepass' /apps/usr/redis-6.2.4/etc/redis-6389.conf 
requirepass 123456
[root@BJ-HW-VM-17-227 ~]# grep 'masterauth' /apps/usr/redis-6.2.4/etc/redis-6389.conf
# masterauth <master-password>
[root@BJ-HW-VM-17-227 ~]# grep 'cluster-enabled yes' /apps/usr/redis-6.2.4/etc/redis-6389.conf
# cluster-enabled yes
[root@BJ-HW-VM-17-227 ~]# grep 'cluster-config-file nodes-6379.conf' /apps/usr/redis-6.2.4/etc/redis-6389.conf
# cluster-config-file nodes-6379.conf
[root@BJ-HW-VM-17-227 ~]# grep 'cluster-require-full-coverage no' /apps/usr/redis-6.2.4/etc/redis-6389.conf


#命令
sed -i -e '/masterauth/a masterauth 123456' -e '/# cluster-enabled yes/a cluster-enabled yes' -e '/# cluster-config-file nodes-6379.conf/a cluster-config-file nodes-6389.conf' -e '/# cluster-require-full-coverage yes/c cluster-require-full-coverage no' /apps/usr/redis-6.2.4/etc/redis-6389.conf

sed -i -e '/masterauth/a masterauth 123456' -e '/# cluster-enabled yes/a cluster-enabled yes' -e '/# cluster-config-file nodes-6379.conf/a cluster-config-file nodes-6390.conf' -e '/# cluster-require-full-coverage yes/c cluster-require-full-coverage no' /apps/usr/redis-6.2.4/etc/redis-6390.conf

#查看最终修改，每台主机都是如此配置

[root@BJ-HW-VM-17-227 ~]# grep '^bind' /apps/usr/redis-6.2.4/etc/redis-6389.conf 
bind 0.0.0.0 -::1
[root@BJ-HW-VM-17-227 ~]# grep '^requirepass' /apps/usr/redis-6.2.4/etc/redis-6389.conf
requirepass 123456
[root@BJ-HW-VM-17-227 ~]# grep 'masterauth' /apps/usr/redis-6.2.4/etc/redis-6389.conf
# masterauth <master-password>
masterauth 123456
[root@BJ-HW-VM-17-227 ~]# grep 'cluster-enabled yes' /apps/usr/redis-6.2.4/etc/redis-6389.conf
# cluster-enabled yes
cluster-enabled yes
[root@BJ-HW-VM-17-227 ~]# grep 'cluster-config-file nodes-6389.conf' /apps/usr/redis-6.2.4/etc/redis-6389.conf
cluster-config-file nodes-6389.conf
[root@BJ-HW-VM-17-227 ~]#  grep 'cluster-require-full-coverage no' /apps/usr/redis-6.2.4/etc/redis-6389.conf
cluster-require-full-coverage no


#启动主机：
systemctl start redis-6389.service
systemctl enable  redis-6389.service

systemctl start redis-6390.service
systemctl enable  redis-6390.service

5、执行meet操作实现相互通信（由于后期关的老集群，跟老集群冲突，省略此步）

#在任一节点上和其他所有节点进行meet通信
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 --no-auth-warning cluster meet 10.8.17.227 6389
Warning: AUTH failed
OK
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 --no-auth-warning cluster meet 10.8.17.227 6390
Warning: AUTH failed
OK
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 --no-auth-warning cluster meet 10.8.17.228 6389
Warning: AUTH failed
OK
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 --no-auth-warning cluster meet 10.8.17.228 6390
Warning: AUTH failed
OK
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 --no-auth-warning cluster meet 10.8.17.228 6389
Warning: AUTH failed
OK
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 --no-auth-warning cluster meet 10.8.17.228 6390
Warning: AUTH failed
OK

#可以看到所有节点之间可以相互连接通信

227:6389
228:6389

227:6390
228:6390

[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 -p 6389 --no-auth-warning cluster nodes
1ca3fc16d16179e34518ebed2b75ab38b9b034c6 10.8.17.228:6389@16389 master - 0 1641350239862 18 connected
799277c4bbcccd5a3b22d5ee70d23908caad53a7 10.8.17.229:6380@16380 slave,fail 4266d0bff62835ba10fcdf382f3b06d87ce71223 1641349469828 1641349466000 3 disconnected
bbe45025edb35b001b91242e6740a4bd9b64bb56 10.8.17.227:6379@16379 slave,fail af7e0b88aedc374feafb0f1a76aac484163ddd13 1641349242521 1641349236000 14 disconnected
e6b2c320af859abd92a9db04aba5aec26ef073d2 10.8.17.227:6389@16389 myself,master - 0 1641350227000 0 connected
4af87cd5898e209b3108d01962fc20af037ce069 10.8.17.228:6390@16390 master - 0 1641350238000 19 connected
4266d0bff62835ba10fcdf382f3b06d87ce71223 10.8.17.228:6379@16379 master,fail - 1641349389224 1641349385200 3 disconnected 5461-10922
35732a2dab1f71ae89418c3bfcafb65787204ce9 10.8.17.227:6390@16390 master - 0 1641350235000 17 connected
1fb5fbfb58b5e54ac2b78f23db1f041b5109994c 10.8.17.229:6379@16379 master,fail - 1641349464789 1641349461000 20 disconnected 10923-16383
0d9029356d1c57082d42e2a47dd95abd4b45dd27 10.8.17.227:6380@16380 master,fail - 1641349251563 1641349244000 16 disconnected
af7e0b88aedc374feafb0f1a76aac484163ddd13 10.8.17.228:6380@16380 master,fail - 1641349395266 1641349390000 14 disconnected 0-5460

#发现有问题，与老集群接入冲突
#不要用meet通信加集群方式
#下面删除meet通信加集群方式，用create cluster集群方式

[root@BJ-HW-VM-17-227 ~]# cd /apps/usr/redis-6.2.4/data/
[root@BJ-HW-VM-17-227 data]# ls
nodes-6389.conf  nodes-6390.conf
[root@BJ-HW-VM-17-227 data]# rm -rf *.conf
[root@BJ-HW-VM-17-227 data]# ls
[root@BJ-HW-VM-17-227 data]# systemctl restart redis-6389.service 
[root@BJ-HW-VM-17-227 data]# systemctl restart redis-6390.service 
[root@BJ-HW-VM-17-227 data]# cd
[root@BJ-HW-VM-17-227 ~]# redis-cli -h 10.8.17.227 -a 123456 -p 6389 --no-auth-warning cluster nodes
b5c97db475e4478ebc3d5de5f017f233ec9d0241 :6389@16389 myself,master - 0 0 0 connected

6、更改使用create cluster创建集群

#注意：为了使用主从节点不落在同一台机器上，使用如下命令：每台ip+port交叉（没有找到命令指定主从节点的关系的方法）

#我下面的没使用交叉的命令，注意交叉的！！

#命令redis-cli的选项  --cluster-replicas 1 表示每个master对应一个slave节点

# redis-cli -a 123456 --cluster create 10.8.17.227:6389 10.8.17.227:6390 10.8.17.228:6389 10.8.17.228:6390 10.8.17.229:6389 10.8.17.229:6390 --cluster-replicas 1

[root@BJ-HW-VM-17-227 ~]# redis-cli -a 123456 --cluster create 10.8.17.227:6389 10.8.17.227:6390 10.8.17.228:6389 10.8.17.228:6390 10.8.17.229:6389 10.8.17.229:6390 --cluster-replicas 1
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 10.8.17.228:6390 to 10.8.17.227:6389
Adding replica 10.8.17.229:6390 to 10.8.17.228:6389
Adding replica 10.8.17.227:6390 to 10.8.17.229:6389
M: b5c97db475e4478ebc3d5de5f017f233ec9d0241 10.8.17.227:6389
   slots:[0-5460] (5461 slots) master
S: 9c2004f27fd3bcb221c7747142c18d3fd36daaf9 10.8.17.227:6390
   replicates ea87f616810499acd9a6ce5d0965db537b144e29
M: 3b50a208b20d55f739ef2005fb8b8191b6c8852c 10.8.17.228:6389
   slots:[5461-10922] (5462 slots) master
S: f34a68242fa00bcdeb2bc417888a18a1d514b51d 10.8.17.228:6390
   replicates b5c97db475e4478ebc3d5de5f017f233ec9d0241
M: ea87f616810499acd9a6ce5d0965db537b144e29 10.8.17.229:6389
   slots:[10923-16383] (5461 slots) master
S: bf0c838ea285063428b211804af5add43c61fc32 10.8.17.229:6390
   replicates 3b50a208b20d55f739ef2005fb8b8191b6c8852c
Can I set the above configuration? (type 'yes' to accept): yes #输入yes自动创建集群
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
.
>>> Performing Cluster Check (using node 10.8.17.227:6389)
M: b5c97db475e4478ebc3d5de5f017f233ec9d0241 10.8.17.227:6389   #带M的为master
   slots:[0-5460] (5461 slots) master    #已经分配的槽位
   1 additional replica(s)
M: 3b50a208b20d55f739ef2005fb8b8191b6c8852c 10.8.17.228:6389
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
M: ea87f616810499acd9a6ce5d0965db537b144e29 10.8.17.229:6389
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: f34a68242fa00bcdeb2bc417888a18a1d514b51d 10.8.17.228:6390
   slots: (0 slots) slave
   replicates b5c97db475e4478ebc3d5de5f017f233ec9d0241
S: 9c2004f27fd3bcb221c7747142c18d3fd36daaf9 10.8.17.227:6390
   slots: (0 slots) slave
   replicates ea87f616810499acd9a6ce5d0965db537b144e29
S: bf0c838ea285063428b211804af5add43c61fc32 10.8.17.229:6390
   slots: (0 slots) slave
   replicates 3b50a208b20d55f739ef2005fb8b8191b6c8852c
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.   #所有槽位（16384个）分配完成

#观察以上结果，可以看到3组master/slave
master：10.8.17.227:6389---slave：10.8.17.227:6390
master：10.8.17.228:6389---slave：10.8.17.228:6390
master：10.8.17.229:6389---slave：10.8.17.229:6390

7、查看集群

#查看主从状态
[root@BJ-HW-VM-17-227 ~]# redis-cli -a 123456 -p 6389 info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.8.17.228,port=6390,state=online,offset=350,lag=0
master_failover_state:no-failover
master_replid:349367af454d7e6a201693c67733e306d9cbbb01
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:350
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:350


[root@BJ-HW-VM-17-227 ~]#  redis-cli -a 123456 -p 6389 --no-auth-warning cluster nodes
b5c97db475e4478ebc3d5de5f017f233ec9d0241 10.8.17.227:6389@16389 myself,master - 0 1641351147000 1 connected 0-5460
3b50a208b20d55f739ef2005fb8b8191b6c8852c 10.8.17.228:6389@16389 master - 0 1641351149641 3 connected 5461-10922
ea87f616810499acd9a6ce5d0965db537b144e29 10.8.17.229:6389@16389 master - 0 1641351148000 5 connected 10923-16383
f34a68242fa00bcdeb2bc417888a18a1d514b51d 10.8.17.228:6390@16390 slave b5c97db475e4478ebc3d5de5f017f233ec9d0241 0 1641351148637 1 connected
9c2004f27fd3bcb221c7747142c18d3fd36daaf9 10.8.17.227:6390@16390 slave ea87f616810499acd9a6ce5d0965db537b144e29 0 1641351149000 5 connected
bf0c838ea285063428b211804af5add43c61fc32 10.8.17.229:6390@16390 slave 3b50a208b20d55f739ef2005fb8b8191b6c8852c 0 1641351150645 3 connected

8、开启aof

#注意：AOF模式默认是关闭的，第一次开启AOF后，并重启服务生效后，会因为AOF的优先级高于RDB，而AOF默认没有文件存在，从而导致所有数据丢失
[root@BJ-HW-VM-17-227 ~]# redis-cli -a 123456 -p 6389 -h 10.8.17.227
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.8.17.227:6389> CONFIG GET appendonly
1) "appendonly"
2) "no"
10.8.17.227:6389> CONFIG SET appendonly yes
OK
10.8.17.227:6389> CONFIG GET appendonly
1) "appendonly"
2) "yes"
10.8.17.227:6389> exit
[root@BJ-HW-VM-17-227 ~]# redis-cli -a 123456 -p 6390 -h 10.8.17.227
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.8.17.227:6390> CONFIG GET appendonly
1) "appendonly"
2) "no"
10.8.17.227:6390> CONFIG SET appendonly yes
OK
10.8.17.227:6390> CONFIG GET appendonly
1) "appendonly"
2) "yes"
10.8.17.227:6390> exit
[root@BJ-HW-VM-17-227 ~]#