ruby 解析日志，提取集点请求参数的一个例子

Posted 2021-05-15

# vim: set nu sw=2 sts=2:

require 'zlib'

class LogParser
  class Request
    attr_accessor :path, :method, :ip, :started_at, :end_at, :status, :duration, :parameters, :api_key, :id

    def parameters=(parameters)
      @parameters = parameters
      @parameters.delete 'enjar'
    end
  end

  def initialize(file)
    @file = file
    @timestamp_length = 'I, [2017-12-20T16:43:55.604375 #11335]  INFO -- : [6d72a0b7-f270-4a7e-a01d-8f45e3fafb42] '.length
  end

  def parse!
    current_request = nil

    @file.each_line do |line|
      if line[@timestamp_length, 7] == 'Started'
	current_request = Request.new
	current_request.id = line[51, 36]

	line =~ /Started (\w+) "(\S+)" for (\S+) at (.*)/
	current_request.method = $1
	current_request.ip = $3
	current_request.path = $2
	current_request.started_at = $4
      elsif line[@timestamp_length+2, 10] == 'Parameters'
	line =~ /Parameters: (.*)/
	# begin
	current_request.parameters = $1
	# rescue StandardError, SyntaxError => ex
	#   STDERR.puts " #{@file.path}: [#{current_request.id}] * Error: failed to parse parameters: #{line}"
	# end
      elsif line[@timestamp_length+2, 12] == 'BizUser Load'
	line =~ /\["api_key", "(.*?)"\]/
	current_request.api_key = $1 unless !$1 or $1.empty?
      elsif line[@timestamp_length, 9] == 'Completed'
	# 2017-12-20T16:43:55.604375
	line =~ /(\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}).*Completed (\d+).* in ([\d\.]+ms)/
	current_request.status = $2
	current_request.duration = $3
	current_request.end_at = $1

	if current_request.api_key == '25cPmHGvKF5jVe4e6xObMdr628hUDbAVvori7w9aAH25BnnYAsJdyj8Ua3dhzVK8AuIpIitCEmF8Vy2NMkr7RA' \
	    and current_request.path == '/enjar/collect' \
	    and current_request.status == '422'
	  parameters = eval(current_request.parameters)
	  parameters.delete 'enjar'
	  puts "[#{current_request.started_at}] #{parameters}"
	end
      end
    end
  end
end

def parse_file(filename)
  STDERR.puts " * in file #{filename}"
  if File.exists?(filename)
    file = File.new(filename)
    file = Zlib::GzipReader.new(file) if filename.end_with?(".gz")
    parser = LogParser.new(file)
    parser.parse!
  else
    STDERR.puts " error: #{filename} does not exist"
  end
ensure
  file&.close
end

ARGV.each do |filename|
  parse_file filename
end