利用原生命令手动部署redis cluster

Posted 2021-07-28 y_zilong

1、在所有节点安装redis并启动cluster功能

#在所有6个节点上都执行下面相同操作

#redis安装脚本
[root@cent7_6 ~]# cat install_redis.sh 
#!/bin/bash
SRC_DIR=/usr/local/src
COLOR="echo -e \\\\033[01;31m"
END='\\033[0m'
CPUS=`lscpu |awk '/^CPU\\(s\\)/{print $2}'`

URL='http://download.redis.io/releases/'
VERSION=redis-6.2.2
PASSWORD=123456
INSTALL_DIR=/apps/redis

os(){
    if grep -Eqi "CentOS" /etc/issue || grep -Eq "CentOS" /etc/*-release;then
        rpm -q redhat-lsb-core &> /dev/null || { ${COLOR}"安装lsb_release工具"${END};yum -y install  redhat-lsb-core &> /dev/null; }
    fi
    OS_ID=`lsb_release -is`
}

install(){
    if [ ${OS_ID} == "CentOS" ] &> /dev/null;then
        yum  -y install gcc jemalloc-devel || { ${COLOR}"安装软件包失败，请检查网络配置"${END}; exit; }
        rpm -q wget &> /dev/null || yum -y install wget &> /dev/null
    else
        apt -y install make gcc libjemalloc-dev  || { ${COLOR}"安装软件包失败，请检查网络配置"${END}; exit; }
    fi

    cd ${SRC_DIR}
    wget ${URL}${VERSION}.tar.gz || { ${COLOR}"Redis 源码下载失败"${END}; exit; }
    tar xf ${VERSION}.tar.gz
    cd ${VERSION}
    make -j ${CPUS} PREFIX=${INSTALL_DIR} install && ${COLOR}"Redis 编译安装完成"${END} || { ${COLOR}"Redis 编译安装失败"${END};exit ; }
    ln -s ${INSTALL_DIR}/bin/redis-*  /usr/bin/
    mkdir -p ${INSTALL_DIR}/{etc,log,data,run}
    cp redis.conf  ${INSTALL_DIR}/etc/
    sed -i -e 's/bind 127.0.0.1.*/bind 0.0.0.0/'  -e "/# requirepass/a requirepass ${PASSWORD}"  -e "/^dir .*/c dir ${INSTALL_DIR}/data/"  -e "/logfile .*/c logfile ${INSTALL_DIR}/log/redis-6379.log"  -e  "/^pidfile .*/c  pidfile ${INSTALL_DIR}/run/redis_6379.pid" ${INSTALL_DIR}/etc/redis.conf

    if id redis &> /dev/null ;then
        ${COLOR}"Redis 用户已存在"${END}
    else
        useradd -r -s /sbin/nologin redis
        ${COLOR}"Redis 用户创建成功"${END}
    fi

    chown -R redis.redis ${INSTALL_DIR}
    cat >> /etc/sysctl.conf <<-EOF
net.core.somaxconn = 1024
vm.overcommit_memory = 1
EOF
    sysctl -p
    if [ ${OS_ID} == "CentOS" ] &> /dev/null;then
        echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.d/rc.local
        chmod +x /etc/rc.d/rc.local
    else
        cat >> /lib/systemd/system/rc-local.service <<-EOF

[Install]
WantedBy=multi-user.target
EOF
        echo '#!/bin/bash' > /etc/rc.local
        echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local
        chmod +x /etc/rc.local
    fi

    cat > /lib/systemd/system/redis.service <<-EOF
[Unit]
Description=Redis persistent key-value database
After=network.target

[Service]
ExecStart=${INSTALL_DIR}/bin/redis-server ${INSTALL_DIR}/etc/redis.conf --supervised systemd
ExecStop=/bin/kill -s QUIT \\$MAINPID
#Type=notify
User=redis
Group=redis
RuntimeDirectory=redisRuntimeDirectoryMode=0755

[Install]
WantedBy=multi-user.target
EOF
    systemctl daemon-reload
    systemctl enable --now  redis &> /dev/null && ${COLOR}"Redis 服务启动成功,Redis信息如下:"${END} || { ${COLOR}"Redis 启动失败"${END};exit; }
    sleep 1
    redis-cli -a ${PASSWORD} INFO Server 2> /dev/null
}

main(){
    os
    install
}

main

##手动修改配置文件
[root@cent7_6 ~]# vim /apps/redis/etc/redis.conf 
bind 0.0.0.0
requirepass 123456
masterauth 123456    #后期的master和slave主从复制
cluster-enabled yes  #开启集群，开启后redis进程会有cluster标识
cluster-config-file nodes-6379.conf   #集群状态文件，记录主从关系及slot范围信息，由redis cluster集群自动创建和维护
cluster-require-full-coverage no   #设为no可以防止一个节点不可用导致整个cluster不可用

[root@cent7_16 ~]# sed -i -e '/masterauth/a masterauth 123456' -e '/# cluster-enabled yes/a cluster-enabled yes' -e '/# cluster-config-file nodes-6379.conf/a cluster-config-file nodes-6379.conf' -e '/# cluster-require-full-coverage yes/c cluster-require-full-coverage no' /apps/redis/etc/redis.conf

##
[root@cent7_6 ~]# grep '^bind' /apps/redis/etc/redis.conf 
bind 0.0.0.0
[root@cent7_6 ~]# grep '^requirepass' /apps/redis/etc/redis.conf 
requirepass 123456

[root@cent7_6 ~]# sed -i -e '/masterauth/a masterauth 123456' /apps/redis/etc/redis.conf 
[root@cent7_6 ~]# grep 'masterauth' /apps/redis/etc/redis.conf 
# masterauth <master-password>
masterauth 123456

[root@cent7_6 ~]# sed -i -e '/# cluster-enabled yes/a cluster-enabled yes' /apps/redis/etc/redis.conf 
[root@cent7_6 ~]# grep 'cluster-enabled yes' /apps/redis/etc/redis.conf 
# cluster-enabled yes
cluster-enabled yes

[root@cent7_6 ~]# sed -i -e '/# cluster-config-file nodes-6379.conf/a cluster-config-file nodes-6379.conf' /apps/redis/etc/redis.conf 
[root@cent7_6 ~]# grep 'cluster-config-file nodes-6379.conf' /apps/redis/etc/redis.conf 
# cluster-config-file nodes-6379.conf
cluster-config-file nodes-6379.conf

[root@cent7_6 ~]# sed -i -e '/# cluster-require-full-coverage yes/c cluster-require-full-coverage no' /apps/redis/etc/redis.conf 
[root@cent7_6 ~]# grep 'cluster-require-full-coverage no' /apps/redis/etc/redis.conf 
cluster-require-full-coverage no

[root@cent7_6 ~]# systemctl enable --now redis

2、执行meet操作实现相互通信

在任一节点上和其他所有节点进行meet通信
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster meet 10.0.0.26 6379
OK
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster meet 10.0.0.36 6379
OK
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster meet 10.0.0.46 6379
OK
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster meet 10.0.0.56 6379
OK

#可以看到所有节点之间可以相互连接通信
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster nodes
62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9 10.0.0.6:6379@16379 myself,master - 0 1625305420000 1 connected
d5bbb4c42075f047cf7310ab073226b6dcca0b3a 10.0.0.46:6379@16379 master - 0 1625305422358 0 connected
a6a5afa5ff8db2a490677cf88cb5832b7796ebbc 10.0.0.16:6379@16379 master - 0 1625305420000 4 connected
74ea213bfd557b1dd640bf05d85f9881fe021200 10.0.0.56:6379@16379 master - 0 1625305421000 5 connected
5b01766d8185f7440c8973e2d243ca163fdd28e4 10.0.0.36:6379@16379 master - 0 1625305420000 2 connected
04429e1e032ae10bb1628af49fb2d38af4d580ff 10.0.0.26:6379@16379 master - 0 1625305421350 3 connected

注意：myid不要冲突，自动生成必须唯一，修改此值需重启
注意：节点通信一会又断开，记得防火墙要关闭
[root@cent7_6 ~]# systemctl stop firewalld

#由于没有槽位无法创建key
[root@cent7_6 ~]# redis-cli -a 123456 --no-auth-warning set name yzl
(error) CLUSTERDOWN Hash slot not served

#查看当前状态
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster info
cluster_state:fail
cluster_slots_assigned:0       #无槽位分配置
cluster_slots_ok:0
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:0              #无集群成员
cluster_current_epoch:5
cluster_my_epoch:1
cluster_stats_messages_ping_sent:525
cluster_stats_messages_pong_sent:479
cluster_stats_messages_meet_sent:5
cluster_stats_messages_sent:1009
cluster_stats_messages_ping_received:479
cluster_stats_messages_pong_received:530
cluster_stats_messages_received:1009

3、为各个master节点指派槽位范围

#创建添加槽位的脚本
[root@cent7_6 ~]# cat addslot.sh 
#!/bin/bash
host=$1
port=$2
start=$3
end=$4
pass=123456
for slot in `seq ${start} ${end}`;do
   echo slot:$slot
   redis-cli -h ${host} -p $port -a ${pass} --no-auth-warning cluster addslots ${slot}
done

#为三个master分配槽位，共16384/3，平均每个master分配5461个槽位
[root@cent7_6 ~]# bash addslot.sh 10.0.0.6 6379 0 5461
[root@cent7_16 ~]# bash addslot.sh 10.0.0.16 6379 5462 10922
[root@cent7_26 ~]# bash addslot.sh 10.0.0.26 6379 10923 16383

#当第一个master分配完槽位后，可以看到下面信息
[root@cent7_6 ~]# redis-cli -a 123456 --no-auth-warning cluster info
cluster_state:ok
cluster_slots_assigned:5462      #分配槽位数
cluster_slots_ok:5462
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:1               #加入集群
cluster_current_epoch:5
cluster_my_epoch:1
cluster_stats_messages_ping_sent:1035
cluster_stats_messages_pong_sent:957
cluster_stats_messages_meet_sent:5
cluster_stats_messages_sent:1997
cluster_stats_messages_ping_received:957
cluster_stats_messages_pong_received:1040
cluster_stats_messages_received:1997

#当第一个master分配完槽位后，可以看到下面信息
[root@cent7_6 ~]# redis-cli -a 123456 --no-auth-warning cluster nodes
62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9 10.0.0.6:6379@16379 myself,master - 0 1625310709000 1 connected 0-5461
d5bbb4c42075f047cf7310ab073226b6dcca0b3a 10.0.0.46:6379@16379 master - 0 1625310712139 0 connected
a6a5afa5ff8db2a490677cf88cb5832b7796ebbc 10.0.0.16:6379@16379 master - 0 1625310711000 4 connected 
74ea213bfd557b1dd640bf05d85f9881fe021200 10.0.0.56:6379@16379 master - 0 1625310713148 5 connected
5b01766d8185f7440c8973e2d243ca163fdd28e4 10.0.0.36:6379@16379 master - 0 1625310709112 2 connected
04429e1e032ae10bb1628af49fb2d38af4d580ff 10.0.0.26:6379@16379 master - 0 1625310711129 3 connected 

#当所有三个节点都分配槽位后可以创建key
[root@cent7_6 ~]# redis-cli -a 123456 --no-auth-warning set name yzl
(error) MOVED 5798 10.0.0.16:6379
[root@cent7_6 ~]# redis-cli -h 10.0.0.16 -a 123456 --no-auth-warning set name yzl
OK
[root@cent7_6 ~]# redis-cli -h 10.0.0.16 -a 123456 --no-auth-warning get name
"yzl"

#当所有的三个master分配完槽位后，可以看到下面信息，所有节点都是master
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster nodes
62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9 10.0.0.6:6379@16379 myself,master - 0 1625311375000 1 connected 0-5461
d5bbb4c42075f047cf7310ab073226b6dcca0b3a 10.0.0.46:6379@16379 master - 0 1625311379170 0 connected
a6a5afa5ff8db2a490677cf88cb5832b7796ebbc 10.0.0.16:6379@16379 master - 0 1625311377000 4 connected 5462-10922
74ea213bfd557b1dd640bf05d85f9881fe021200 10.0.0.56:6379@16379 master - 0 1625311378161 5 connected
5b01766d8185f7440c8973e2d243ca163fdd28e4 10.0.0.36:6379@16379 master - 0 1625311376142 2 connected
04429e1e032ae10bb1628af49fb2d38af4d580ff 10.0.0.26:6379@16379 master - 0 1625311377151 3 connected 10923-16383

#当所有的三个master分配完槽位后，可以看到下面信息
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3             #三个成员
cluster_current_epoch:5
cluster_my_epoch:1
cluster_stats_messages_ping_sent:6023
cluster_stats_messages_pong_sent:5578
cluster_stats_messages_meet_sent:5
cluster_stats_messages_sent:11606
cluster_stats_messages_ping_received:5578
cluster_stats_messages_pong_received:6028
cluster_stats_messages_received:11606

4、指定各个节点的主从关系

#通过上面cluster nodes 查看master的ID信息，执行下面操作，将对应的slave指定相应的master节点，实现三对主从节点
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster nodes
62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9 10.0.0.6:6379@16379 myself,master - 0 1625311882000 1 connected 0-5461
d5bbb4c42075f047cf7310ab073226b6dcca0b3a 10.0.0.46:6379@16379 master - 0 1625311884815 0 connected
a6a5afa5ff8db2a490677cf88cb5832b7796ebbc 10.0.0.16:6379@16379 master - 0 1625311883805 4 connected 5462-10922
74ea213bfd557b1dd640bf05d85f9881fe021200 10.0.0.56:6379@16379 master - 0 1625311883000 5 connected
5b01766d8185f7440c8973e2d243ca163fdd28e4 10.0.0.36:6379@16379 master - 0 1625311882797 2 connected
04429e1e032ae10bb1628af49fb2d38af4d580ff 10.0.0.26:6379@16379 master - 0 1625311881785 3 connected 10923-16383

#将10.0.0.36指定10.0.0.6的ID做为其从节点
[root@cent7_6 ~]# redis-cli -h 10.0.0.36 -a 123456 --no-auth-warning cluster replicate 62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9
OK

#将10.0.0.46指定10.0.0.16的ID做为其从节点
[root@cent7_6 ~]# redis-cli -h 10.0.0.46 -a 123456 --no-auth-warning cluster replicate a6a5afa5ff8db2a490677cf88cb5832b7796ebbc
OK

#将10.0.0.56指定10.0.0.26的ID做为其从节点
[root@cent7_6 ~]# redis-cli -h 10.0.0.56 -a 123456 --no-auth-warning cluster replicate 04429e1e032ae10bb1628af49fb2d38af4d580ff
OK

#所有三组主从节点创建成功后，可以看到最终结果
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster nodes
62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9 10.0.0.6:6379@16379 myself,master - 0 1625312221000 1 connected 0-5461
d5bbb4c42075f047cf7310ab073226b6dcca0b3a 10.0.0.46:6379@16379 slave a6a5afa5ff8db2a490677cf88cb5832b7796ebbc 0 1625312220000 4 connected
a6a5afa5ff8db2a490677cf88cb5832b7796ebbc 10.0.0.16:6379@16379 master - 0 1625312221887 4 connected 5462-10922
74ea213bfd557b1dd640bf05d85f9881fe021200 10.0.0.56:6379@16379 slave 04429e1e032ae10bb1628af49fb2d38af4d580ff 0 1625312220000 3 connected
5b01766d8185f7440c8973e2d243ca163fdd28e4 10.0.0.36:6379@16379 slave 62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9 0 1625312219868 1 connected
04429e1e032ae10bb1628af49fb2d38af4d580ff 10.0.0.26:6379@16379 master - 0 1625312220878 3 connected 10923-16383

[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:5
cluster_my_epoch:1
cluster_stats_messages_ping_sent:7046
cluster_stats_messages_pong_sent:6541
cluster_stats_messages_meet_sent:5
cluster_stats_messages_sent:13592
cluster_stats_messages_ping_received:6541
cluster_stats_messages_pong_received:7051
cluster_stats_messages_received:13592

[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning info replication
# Replication
role:master
connected_slaves:1
slave0:ip=10.0.0.36,port=6379,state=online,offset=784,lag=1
master_failover_state:no-failover
master_replid:26c3dd43e9c141435990237462eeac636bde0f9c
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:784
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:784

[root@cent7_6 ~]# redis-cli -h 10.0.0.16 -a 123456 --no-auth-warning info replication
# Replication
role:master
connected_slaves:1
slave0:ip=10.0.0.46,port=6379,state=online,offset=826,lag=1
master_failover_state:no-failover
master_replid:a0514d3afd3124fcbc7590fd2f517b6a96d2f820
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:826
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:826

[root@cent7_6 ~]# redis-cli -h 10.0.0.26 -a 123456 --no-auth-warning info replication
# Replication
role:master
connected_slaves:1
slave0:ip=10.0.0.56,port=6379,state=online,offset=630,lag=1
master_failover_state:no-failover
master_replid:e4db8467cd3611246348e824f81fe48ab9f1e9f8
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:630
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:630

[root@cent7_6 ~]# redis-cli -h 10.0.0.36 -a 123456 --no-auth-warning info replication
# Replication
role:slave
master_host:10.0.0.6
master_port:6379
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:1134
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:26c3dd43e9c141435990237462eeac636bde0f9c
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1134
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:1134

#查看主从节关系及槽位信息
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning cluster slots
1) 1) (integer) 0
   2) (integer) 5461
   3) 1) "10.0.0.6"
      2) (integer) 6379
      3) "62aa9c9ad5b7e5395ca40360c5bb46f1d35eccc9"
   4) 1) "10.0.0.36"
      2) (integer) 6379
      3) "5b01766d8185f7440c8973e2d243ca163fdd28e4"
2) 1) (integer) 5462
   2) (integer) 10922
   3) 1) "10.0.0.16"
      2) (integer) 6379
      3) "a6a5afa5ff8db2a490677cf88cb5832b7796ebbc"
   4) 1) "10.0.0.46"
      2) (integer) 6379
      3) "d5bbb4c42075f047cf7310ab073226b6dcca0b3a"
3) 1) (integer) 10923
   2) (integer) 16383
   3) 1) "10.0.0.26"
      2) (integer) 6379
      3) "04429e1e032ae10bb1628af49fb2d38af4d580ff"
   4) 1) "10.0.0.56"
      2) (integer) 6379
      3) "74ea213bfd557b1dd640bf05d85f9881fe021200"

5、验证redis cluster访问

#指定选项 -c 表示以集群方式连接
[root@cent7_6 ~]# redis-cli -c -h 10.0.0.6 -a 123456 --no-auth-warning set name yyy
OK
[root@cent7_6 ~]# redis-cli -c -h 10.0.0.6 -a 123456 --no-auth-warning get name
"yyy"

#非集群方式连接
[root@cent7_6 ~]# redis-cli -h 10.0.0.6 -a 123456 --no-auth-warning get name
(error) MOVED 5798 10.0.0.16:6379
[root@cent7_6 ~]# redis-cli -h 10.0.0.16 -a 123456 --no-auth-warning get name
"yyy"
[root@cent7_6 ~]# redis-cli -h 10.0.0.26 -a 123456 --no-auth-warning get name
(error) MOVED 5798 10.0.0.16:6379