监控企业级服务架构搭建

Posted 2021-10-05 我的紫霞辣辣

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了监控企业级服务架构搭建相关的知识，希望对你有一定的参考价值。

数据库集群(keepalived+mysql双主架构)

ip地址	作用
192.168.15.210	VIP，Zabbix集群连接MySQL集群专⽤
192.168.15.204 (zabbix04)	MySQL集群本地IP，Local IP
192.168.15.205 (zabbix05)	MySQL集群本地IP，Local IP

数据库双主架构

下载包，安装数据库

# 数据库客户端程序
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-client-8.0.23-14.1.el7.x86_64.rpm	

# 数据库主程序
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-server-8.0.23-14.1.el7.x86_64.rpm

# 数据库主程序依赖
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-shared-8.0.23-14.1.el7.x86_64.rpm

# 数据库主程序依赖
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-shared-compat-8.0.23-14.1.el7.x86_64.rpm

# 安装数据库
yum install -y *.rpm

修改配置文件

vim /etc/my.cnf
[mysqld]
...
server-id=51
binlog_format=row
log-bin=mysql-bin
skip-name-resolve 		# 跳过域名解析（非必须）
log-slave-updates=1 	# slave更新是否记入日志（5.6必须的）
relay_log_purge = 0		 # 关闭relay_log自动清除功能，保障故障时的数据一致

vim /etc/my.cnf
[mysqld]
...
server-id=52
binlog_format=row
log-bin=mysql-bin
skip-name-resolve 		# 跳过域名解析（非必须）
log-slave-updates=1 	# slave更新是否记入日志（5.6必须的）
relay_log_purge = 0		 # 关闭relay_log自动清除功能，保障故障时的数据一致

数据库初始化

mysqld --defaults-file=/etc/my.cnf --datadir=/data --user=mysql --initialize-insecure
systemctl start mysqld

# 查看数据库密码
cat /var/log/mysqld.log | grep password		

# 登陆数据库
mysql -uroot -p(密码)

# 重新设置数据库管理员密码
ALTER USER 'root'@'localhost' IDENTIFIED BY 'Na123***' PASSWORD EXPIRE NEVER;

# 创建主从库复制用户
create user 'rep'@'192.168.15.205' identified by 'Na123***';
grant replication slave on *.* to rep@192.168.15.205;

# 刷新授权表
flush privileges;
exit;

# 将mysqld设置为开机自启
systemctl enable mysqld

mysqld --defaults-file=/etc/my.cnf --datadir=/data --user=mysql --initialize-insecure
systemctl start mysqld

# 查看数据库密码
cat /var/log/mysqld.log | grep password		

# 登陆数据库
mysql -uroot -p(密码)

# 重新设置数据库管理员密码
ALTER USER 'root'@'localhost' IDENTIFIED BY 'Na123***' PASSWORD EXPIRE NEVER;

# 创建主从库复制用户
create user 'rep'@'192.168.15.204' identified by 'Na123***';
grant replication slave on *.* to rep@192.168.15.204;

# 刷新授权表
flush privileges;
exit;

# 将mysqld设置为开机自启
systemctl enable mysqld

配置双主架构

mysql -uroot -p'Na123***'

mysql> show master status \\G
*************************** 1. row ***************************
             File: mysql-bin.000002
         Position: 1221
     Binlog_Do_DB: 
 Binlog_Ignore_DB: 
Executed_Gtid_Set: 
1 row in set (0.00 sec)

change master to 
get_master_public_key=1,
master_host='192.168.15.204',
master_user='rep',
master_password='Na123***',
master_log_file='mysql-bin.000002',
master_log_pos=1221;


start slave;

# 查看主从复制服务配置是否完成
mysql> show slave status\\G
#      Slave_IO_Running: Yes			IO线程
#      Slave_SQL_Running: Yes			SQL线程

mysql> show master status \\G
*************************** 1. row ***************************
             File: mysql-bin.000002
         Position: 1454
     Binlog_Do_DB: 
 Binlog_Ignore_DB: 
Executed_Gtid_Set: 
1 row in set (0.00 sec)

change master to 
get_master_public_key=1,
master_host='192.168.15.205',
master_user='rep',
master_password='Na123***',
master_log_file='mysql-bin.000002',
master_log_pos=1454;


start slave;

# 查看主从复制服务配置是否完成
mysql> show slave status\\G
#      Slave_IO_Running: Yes			IO线程
#      Slave_SQL_Running: Yes			SQL线程

创建zabbix服务相关的数据

create user zabbix@"%" identified by 'Na123***';

create database zabbix character set utf8 collate utf8_bin;

flush privileges;

系统优化

# 关闭防火墙和selinux

# linux资源限制设置
vim /etc/security/limits.conf
# soft是一个警告值，而hard则是一个真正意义的阀值，超过就会报错
* soft nproc 65535		 # 任何用户可以打开的最大进程数 
* hard nproc 65535
* soft nofile 102400	    # 任何用户可以打开的最大的文件描述符数量，默认1024，这里的数值会限制tcp连接
* hard nofile 102400

# 统一时区
timedatectl set-timezone "Asia/Shanghai"

# 重启数据库
reboot

数据库高可用

高可用工具

yum install -y keepalived
systemctl enable keepalived

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
 notification_email {
 	sysadmin@firewall.loc
 }
 notification_email_from localhost
 smtp_server 127.0.0.1
 smtp_connect_timeout 30
 router_id MySQL_Cluster
}

vrrp_instance MySQL_HA {
 state BACKUP
 interface eth0
 virtual_router_id 31
 priority 100
 advert_int 1
 nopreempt
 authentication {
    auth_type PASS
 	auth_pass 1111
 }
 
 virtual_ipaddress {
 	192.168.15.210 				# 虚拟IP
 }
}

virtual_server 192.168.15.210 3306 {
 delay_loop 6
 persistence_timeout 30
 protocol TCP
 
 real_server 192.168.15.204 3306 {
 	notify_down /etc/keepalived.sh
 	TCP_CHECK {
 		connect_port 3306
 		connect_timeout 3
 		nb_get_retry 1
 		delay_before_retry 1
 		}
 	}
}

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
 notification_email {
 	sysadmin@firewall.loc
 }
 notification_email_from localhost
 smtp_server 127.0.0.1
 smtp_connect_timeout 30
 router_id MySQL_Cluster
}

vrrp_instance MySQL_HA {
 state BACKUP
 interface eth0
 virtual_router_id 31
 priority 90
 nopreempt
 advert_int 1
 authentication {
 	auth_type PASS
 	auth_pass 1111
 }
 virtual_ipaddress {
 	192.168.15.210
 }
}

virtual_server 192.168.15.210 3306 {
 delay_loop 6
 persistence_timeout 30
 protocol TCP
 
 real_server 192.168.15.205 3306 {
 	notify_down /etc/keepalived.sh
    TCP_CHECK {
 		connect_port 3306
 		connect_timeout 3
 		nb_get_retry 1
        delay_before_retry 1
 		}
 	}
}

vim /etc/keepalived.sh
#!/bin/bash
systemctl stop keepalived

chmod +x /etc/keepalived.sh

# 重启keepalived并设置开机自启
systemctl restart keepalived
systemctl enable keepalived

测试

关闭虚拟ip地址所在主机的mysqld服务，看一下虚拟ip地址是否会漂移。

Zabbix集群

ip地址	作用
192.168.15.211	VIP，Zabbix对外提供的IP地址，WEB集群通讯⼊⼝IP地址
192.168.15.202(zabbix02)	Zabbix集群节点和Web节点本地IP，Local IP
192.168.15.203(zabbix03)	Zabbix集群节点和Web节点本地IP，Local IP

安装MySQL客户端

# MySQL客户端软件包
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-client-8.0.23-14.1.el7.x86_64.rpm

# MySQL驱动依赖程序包
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-devel-8.0.23-14.1.el7.x86_64.rpm

# MySQL驱动依赖程序包
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-shared-8.0.23-14.1.el7.x86_64.rpm

# MySQL驱动依赖程序包
wget https://downloads.percona.com/downloads/Percona-Server-LATEST/Percona-Server-8.0.23-14/binary/redhat/7/x86_64/percona-server-shared-compat-8.0.23-14.1.el7.x86_64.rpm

# 数据库驱动依赖安装
yum -y install percona-server-devel-8.0.23-14.1.el7.x86_64.rpm percona-server-shared-8.0.23-14.1.el7.x86_64.rpm percona-server-shared-compat-8.0.23-14.1.el7.x86_64.rpm percona-server-client-8.0.23-14.1.el7.x86_64.rpm

Zabbix Server安装

Zabbix 编译安装

# zabbix功能依赖安装
yum install -y libxml2-devel net-snmp-devel libssh2-devel OpenIPMI-devel libevent-devel openldap-devel libcurl-devel gcc gcc-c++ unixODBC-devel

# 下载zabbix5.4.0安装包
wget https://cdn.zabbix.com/zabbix/sources/stable/5.4/zabbix-5.4.0.tar.gz

# 解压
tar -xf zabbix-5.4.0.tar.gz

# 编译安装，激活server和agent所有需要的功能
cd zabbix-5.4.0/
./configure --prefix=/usr/local/zabbix --enable-server --enable-agent --with-net-snmp --with-libcurl --with-libxml2 --with-mysql --with-openipmi --with-ldap --with-ssh2 --with-unixodbc --with-libenvent

# 编译安装
make install

配置文件

两个节点保证完全⼀致

检查如下⽂件 /usr/local/zabbix/etc/zabbix_agentd.conf 是否和下⽂相匹配
检查如下⽂件 /usr/local/zabbix/etc/zabbix_agentd.conf 是否和下⾯相匹配

vim /usr/local/zabbix/etc/zabbix_agentd.conf
LogFile=/tmp/zabbix_agentd.log
Server=192.168.15.211	
ServerActive=192.168.15.211
HostnameItem=system.hostname

vim /usr/local/zabbix/etc/zabbix_server.conf
LogFile=/tmp/zabbix_server.log
PidFile=/tmp/zabbix_server.pid
DBHost=192.168.15.210
DBName=zabbix
DBUser=zabbix
DBPassword=Na123***
StartPollers=31
StartIPMIPollers=1
StartPreprocessors=5
StartPollersUnreachable=11
StartTrappers=11
StartPingers=5
StartDiscoverers=3
StartHTTPPollers=11
StartTimers=5
StartEscalators=5
StartVMwareCollectors=1
StartSNMPTrapper=1
CacheSize=1G
CacheUpdateFrequency=60
StartDBSyncers=4
HistoryCacheSize=2G
HistoryIndexCacheSize=512M
TrendCacheSize=512M
TrendFunctionCacheSize=512M
ValueCacheSize=1G
Timeout=30
AlertScriptsPath=/usr/local/zabbix/alertscripts
ExternalScripts=/usr/local/zabbix/externalscripts
LogSlowQueries=3000
StatsAllowedIP=127.0.0.1,192.168.15.211,192.168.15.202,192.168.15.203

数据初始化

在其中⼀台Zabbix Server上操作即可

cd zabbix-5.4.0/
mysql -uzabbix -p"Na123***" -h 192.168.15.210 zabbix < database/mysql/schema.sql
mysql -uzabbix -p"Na123***" -h 192.168.15.210 zabbix < database/mysql/images.sql
mysql -uzabbix -p"Na123***" -h 192.168.15.210 zabbix < database/mysql/data.sql

表分区

mysql -uzabbix -p"Na123***" -h 192.168.15.210

use zabbix;

DELIMITER $$
CREATE PROCEDURE `partition_create`(SCHEMANAME varchar(64), TABLENAME varchar(64), 
PARTITIONNAME varchar(64), CLOCK int)
BEGIN
 /*
 SCHEMANAME = The DB schema in which to make changes
 TABLENAME = The table with partitions to potentially delete
 PARTITIONNAME = The name of the partition to create
 */
 /*
 Verify that the partition does not already exist
 */
 DECLARE RETROWS INT;
 SELECT COUNT(1) INTO RETROWS
 FROM information_schema.partitions
 WHERE table_schema = SCHEMANAME AND table_name = TABLENAME AND
partition_description >= CLOCK;
 IF RETROWS = 0 THEN
 /*
 1. Print a message indicating that a partition was created.
 2. Create the SQL to create the partition.
 3. Execute the SQL from #2.
 */
 SELECT CONCAT( "partition_create(", SCHEMANAME, ",", TABLENAME, ",", 
PARTITIONNAME, ",", CLOCK, ")" ) AS msg;
 SET @sql = CONCAT( 'ALTER TABLE ', SCHEMANAME, '.', TABLENAME, ' ADD
PARTITION (PARTITION ', PARTITIONNAME, ' VALUES LESS THAN (', CLOCK, '));' );
 PREPARE STMT FROM @sql;
 EXECUTE STMT;
 DEALLOCATE PREPARE STMT;
 END IF;
END$$
DELIMITER ;


DELIMITER $$
CREATE PROCEDURE `partition_drop`(SCHEMANAME VARCHAR(64), TABLENAME VARCHAR(64), 
DELETE_BELOW_PARTITION_DATE BIGINT)
BEGIN
 /*
 SCHEMANAME = The DB schema in which to make changes
 TABLENAME = The table with partitions to potentially delete
 DELETE_BELOW_PARTITION_DATE = Delete any partitions with names that are dates
older than this one (yyyy-mm-dd)
 */
 DECLARE done INT DEFAULT FALSE;
 DECLARE drop_part_name VARCHAR(16);
 /*
 Get a list of all the partitions that are older than the date
 in DELETE_BELOW_PARTITION_DATE. All partitions are prefixed with
 a "p", so use SUBSTRING TO get rid of that character.
 */
 DECLARE myCursor CURSOR FOR
 SELECT partition_name
 FROM information_schema.partitions
 WHERE table_schema = SCHEMANAME AND table_name = TABLENAME AND
CAST(SUBSTRING(partition_name FROM 2) AS UNSIGNED) < DELETE_BELOW_PARTITION_DATE;
 DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
 /*
 Create the basics for when we need to drop the partition. Also, create
 @drop_partitions to hold a comma-delimited list of all partitions that
 should be deleted.
 */
 SET @alter_header = CONCAT("ALTER TABLE ", SCHEMANAME, ".", TABLENAME, " DROP
PARTITION ");
 SET @drop_partitions = "";
 /*
 Start looping through all the partitions that are too old.
 */
 OPEN myCursor;
 read_loop: LOOP
 FETCH myCursor INTO drop_part_name;
 IF done THEN
 LEAVE read_loop;
 END IF;
 SET @drop_partitions = IF(@drop_partitions = "", drop_part_name, 
CONCAT(@drop_partitions, ",", drop_part_name));
 END LOOP;
 IF @drop_partitions != "" THEN
 /*
 1. Build the SQL to drop all the necessary partitions.
 2. Run the SQL to drop the partitions.
 3. Print out the table partitions that were deleted.
 */
 SET @full_sql = CONCAT(@alter_header, @drop_partitions, ";");
 PREPARE STMT FROM @full_sql;
 EXECUTE STMT;
 DEALLOCATE PREPARE STMT;
 SELECT CONCAT(SCHEMANAME, ".", TABLENAME) AS `table`, @drop_partitions AS
`partitions_deleted`;
 ELSE
 /*
 No partitions are being deleted, so print out "N/A" (Not applicable)
to indicate
 that no changes were made.
 */
 SELECT CONCAT(SCHEMANAME, ".", TABLENAME) AS `table`, "N/A" AS
`partitions_deleted`;
 END IF;
END$$
DELIMITER ;


DELIMITER $$
CREATE PROCEDURE `partition_maintenance`(SCHEMA_NAME VARCHAR(32), TABLE_NAME VARCHAR(32), 
KEEP_DATA_DAYS INT, HOURLY_INTERVAL INT, CREATE_NEXT_INTERVALS INT)
BEGIN
 DECLARE OLDER_THAN_PARTITION_DATE VARCHAR(16);
 DECLARE PARTITION_NAME VARCHAR(16);
 DECLARE OLD_PARTITION_NAME VARCHAR(16);
 DECLARE LESS_THAN_TIMESTAMP INT;
 DECLARE CUR_TIME INT;
 CALL partition_verify(SCHEMA_NAME, TABLE_NAME, HOURLY_INTERVAL);
 SET CUR_TIME = UNIX_TIMESTAMP(DATE_FORMAT(NOW(), '%Y-%m-%d 00:00:00'));
 SET @__interval = 1;
 create_loop: LOOP
 IF @__interval > CREATE_NEXT_INTERVALS THEN
 LEAVE create_loop;
 END IF;
 SET LESS_THAN_TIMESTAMP = CUR_TIME + (HOURLY_INTERVAL * @__interval *
3600);
 SET PARTITION_NAME = FROM_UNIXTIME(CUR_TIME + HOURLY_INTERVAL *
(@__interval - 1) * 3600, 'p%Y%m%d%H00');
 IF(PARTITION_NAME != OLD_PARTITION_NAME) THEN
 CALL partition_create(SCHEMA_NAME, TABLE_NAME, PARTITION_NAME, 
LESS_THAN_TIMESTAMP);
 END IF;
 SET @__interval=@__interval+1;
 SET OLD_PARTITION_NAME = PARTITION_NAME;
 END LOOP;
 SET OLDER_THAN_PARTITION_DATE=DATE_FORMAT(DATE_SUB(NOW(), INTERVAL KEEP_DATA_DAYS
DAY), '%Y%m%d0000');
 CALL partition_drop(SCHEMA_NAME, TABLE_NAME, OLDER_THAN_PARTITION_DATE);
END$$
DELIMITER ;


DELIMITER $$
CREATE PROCEDURE `partition_verify`(SCHEMANAME VARCHAR(64), TABLENAME VARCHAR(64), 
HOURLYINTERVAL INT(11))
BEGIN
 DECLARE PARTITION_NAME VARCHAR(16);
 DECLARE RETROWS INT(11);
 DECLARE FUTURE_TIMESTAMP TIMESTAMP;
 /*
 * Check if any partitions exist for the given SCHEMANAME.TABLENAME.
 */
 SELECT COUNT(1) INTO RETROWS
 FROM information_schema.partitions
 WHERE table_schema = SCHEMANAME AND table_name = TABLENAME AND partition_name IS
NULL;
 /*
 * If partitions do not exist, go ahead and partition the table
 */
 IF RETROWS = 1 THEN
 /*
 * Take the current date at 00:00:00 and add HOURLYINTERVAL to it. This
is the timestamp below which we will store values.
 * We begin partitioning based on the beginning of a day. This is
because we don't want to generate a random partition
 * that won't necessarily fall in line with the desired partition naming
(ie: if the hour interval is 24 hours, we could
 * end up creating a partition now named "p201403270600" when all other
partitions will be like "p201403280000").
 */
 SET FUTURE_TIMESTAMP = TIMESTAMPADD(HOUR, HOURLYINTERVAL, 
CONCAT(CURDATE(), " ", '00:00:00'));
 SET PARTITION_NAME = DATE_FORMAT(CURDATE(), 'p%Y%m%d%H00');
 -- Create the partitioning query
 SET @__PARTITION_SQL = CONCAT("ALTER TABLE ", SCHEMANAME, ".",以上是关于监控企业级服务架构搭建的主要内容，如果未能解决你的问题，请参考以下文章 
 项目实战——企业级Zabbix监控实战
 Zabbix运维监控平台快速搭建实录
 linux中级架构初章一
 演进实录｜不同阶段的企业如何搭建监控体系？
 在LNMP架构中搭建zabbix监控服务！！！
 搭建Prometheus平台，你必须考虑的6个因素