安装go-mysql-elasticsearch同步mysql数据到elasticsearch

Posted 2023-03-01 Json2011315

1.安装go

[root@localhost ~]# mkdir -p /usr/local/app/go
[root@localhost ~]# cd /usr/local/app/go/
[root@localhost go]# wget https://golang.google.cn/dl/go1.15.2.linux-amd64.tar.gz
[root@localhost go]# tar -C /usr/local -zxvf go1.15.2.linux-amd64.tar.gz

配置环境变量 vi /etc/profile

[root@localhost go]# vim /etc/profile
#文件追加

export GOROOT=/usr/local/go 
export GOPATH=/usr/local/app/go 
export PATH=$PATH:/usr/local/go/bin

#执行source 
[root@localhost go]# source /etc/profile

其中GOPATH为go的项目目录地址，对应目录内容

编辑hello.go调试

[root@localhost go]# vim hello.go
	package main
	import "fmt" 
	
	func main()  
		var name string = "go" 
		fmt.Println("hello world ", name)
	 

执行（输出内容表示安装成功）：

[root@localhost go]# go run hello.go 
hello world  go

2.修改mysql binlog日志模式

需要值得注意的是，go-mysql-elasticsearch 支持binlog的日志模式为row模式因此建议在配置MySQLbinlog的时候需指定该模式

server_id = 1001
log_bin="mysql-bin"
binlog_format=row

3. go-mysql-elasticsearch安装

3.1安装依赖包

[root@localhost go]# yum -y install gettext-devel openssl-devel perl-CPAN perl-devel zlib-devel
#安装git
[root@localhost go]# yum -y install git

3.2 执行安装

[root@localhost go]# go env -w GOPROXY=https://goproxy.cn
[root@localhost go]# go get github.com/siddontang/go-mysql-elasticsearch
package github.com/siddontang/go-mysql-elasticsearch: no Go files in /usr/local/app/go/src/github.com/siddontang/go-mysql-elasticsearch

#查看$GOPATH变量
[root@localhost go]# echo $GOPATH
/usr/local/app/go

#查看是否下载成功go-mysql-elasticsearch
[root@localhost go]# ls $GOPATH/src/github.com/siddontang/go-mysql-elasticsearch 
clear_vendor.sh  cmd  Dockerfile  elastic  etc  go.mod  go.sum  LICENSE  Makefile  README.md  river

#进入目录执行安装
[root@localhost go]# cd $GOPATH/src/github.com/siddontang/go-mysql-elasticsearch
[root@localhost go-mysql-elasticsearch]# make

4.配置river.toml

[root@localhost go]# pwd
/usr/local/app/go
[root@localhost go]# ls
go1.15.2.linux-amd64.tar.gz  hello.go  pkg  src
[root@localhost go]# vim river.toml

	# MySQL 配置：地址，用户名，密码
	my_addr = "192.168.11.55:3306"
	my_user = "root"
	my_pass = "123456"
	
	# Elasticsearch地址
	es_addr = "192.168.169.120:9212"
	
	# 存储数据的位置
	data_dir = "/var"
	
	# Inner Http status address
	stat_addr = "192.168.169.120:12800"
	stat_path = "/es"
	# pseudo server id like a slave
	server_id = 1001
	# mysql or mariadb
	flavor = "mysql"
	# mysqldump execution path
	# mysqldump = "mysqldump"
	# minimal items to be inserted in one bulk
	bulk_size = 128
	
	# force flush the pending requests if we don't have enough items >= bulk_size
	flush_bulk_time = "200ms"
	
	# Ignore table without primary key
	skip_no_pk_table = false
	
	# elasticsearch 与 mysql 同步时对应的数据库名称
	# mysql的数据源
	[[source]]
	schema = "srm"
	tables = ["goods","news"]
	
	# es 映射的mapping
	[[rule]]
	schema = "srm"
	table = "goods"
	# es的索引名
	index = "srm_goods"
	type = "link_info"
	
	
	[[rule]]
	schema = "srm"
	table = "news"
	# es的索引名
	index = "srm_news"
	type = "link_news"

启动

[root@localhost go]# $GOPATH/src/github.com/siddontang/go-mysql-elasticsearch/bin/go-mysql-elasticsearch -config=/usr/local/app/go/river.toml

5.创建数据库以及数据表添加数据

mysql客户端执行【创建srm数据库】：

SET FOREIGN_KEY_CHECKS=0;

-- ----------------------------
-- Table structure for goods
-- ----------------------------
DROP TABLE IF EXISTS `goods`;
CREATE TABLE `goods` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `goods_name` varchar(30) DEFAULT '',
  `goods_description` varchar(100) DEFAULT '',
  `goods_info` varchar(100) DEFAULT '',
  PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4;

-- ----------------------------
-- Table structure for news
-- ----------------------------
DROP TABLE IF EXISTS `news`;
CREATE TABLE `news` (
  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
  `title` varchar(50) DEFAULT '',
  `description` varchar(100) DEFAULT '',
  PRIMARY KEY (`id`)
) ENGINE=MyISAM AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;


INSERT INTO `goods` VALUES ('1', '商品1', '商品描述', '商品明细');
INSERT INTO `goods` VALUES ('2', '华为mate30', '华为mate30描述', '华为mate30明细');
INSERT INTO `news` VALUES ('1', '新闻1', '新闻描述1');
INSERT INTO `news` VALUES ('2', '新闻2', '新闻描述2');

6.查看es索引数据