logstash将日志写入数据库mysql

Posted 2023-03-13

# 安装logstsh
cd /usr/local/
wget https://artifacts.elastic.co/downloads/logstash/logstash-8.6.2-linux-x86_64.tar.gz
tar -xzvf logstash-8.6.2-linux-x86_64.tar.gz
ln -s logstash-8.6.2-linux-x86_64 logstash
export PATH=$PATH:/usr/local/logstash/jdk/bin >>/etc/profile
source /etc/profile

# 安装logstash插件
/usr/local/logstash/bin/logstash-plugin install logstash-output-analyticdb
/usr/local/logstash/bin/logstash-plugin install logstash-output-jdbc

# 查看
/usr/local/logstash/bin/logstash-plugin list | grep jdbc

# 下载https://mvnrepository.com/artifact/mysql/mysql-connector-java
# 5.1版本支持mysql5.6/5.7；8.0.x版本支持mysql5.8
mkdir /usr/local/logstash/vendor/jar/jdbc
wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar

# 将json格式gamelog日志中table中含有table:login的json日志写入player表中，
# 将table表中table：online的日志写入renshu表中

测试日志gamelog:
"line": "11", "table": "online"
"name": "xc", "uid": "2574433315", "newBee": true, "table": "login"
"name": "xc", "uid": "2574433315", "newBee": false


input 
    file 
        path => "/db/log/gamelog"
        start_position => "beginning"
        stat_interval => "2"
    


filter 
    grok 
    #GREEDYDATA表达式的意思能匹配任意字符串
    # WORD 匹配字符
        match => "message" => "\\[%WORD:table\\],%GREEDYDATA:message"
        overwrite => [ "message" ]
    
    json 
        source => "message"
        skip_on_invalid_json => true
    
    mutate 
        remove_field => ["@version", "@timestamp", "path", "status","tags"]
    


output 
    stdout codec => json_lines
    if [table] == "login" 
        analyticdb 
            driver_jar_path => "/usr/local/logstash/vendor/jar/jdbc/mysql-connector-java-5.1.49.jar"
            driver_class => "com.mysql.jdbc.Driver"
            connection_string => "jdbc:mysql://192.168.10.14:3306/log_db?user=game&password=wg1q2w3e&useUnicode=true&characterEncoding=utf8&useSSL=false"
            statement => ["INSERT INTO player(accountid,nickname) VALUES(?,?)","uid","name"]
        
    

    if [table] == "online" 
        analyticdb 
            driver_jar_path => "/usr/local/logstash/vendor/jar/jdbc/mysql-connector-java-5.1.49.jar"
            driver_class => "com.mysql.jdbc.Driver"
            connection_string => "jdbc:mysql://192.168.10.14:3306/log_db?user=game&password=wg1q2w3e&useUnicode=true&characterEncoding=utf8&useSSL=false"
            statement => ["INSERT INTO renshu(pcu) VALUES(?)","line"]
        
    

启动测试

nohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/logstash_start.conf &>/dev/null &