ruby 在单独的线程中使用多个连接进行http请求的简单类

Posted 2021-05-15

require "net/http"

#   fetcher = FeedCrawler::URLFetcher.new("example.com")
#   paths = ["/foo", "/bar", "/baz", "/qux"]
#   results = fetcher.fetch(paths)
#   results.each do |result|
#     puts result
#   end
#
class URLFetcher
  class WorkRequest
    def initialize(input)
      @input = input
      @mutex = Mutex.new
    end

    def process
      @mutex.synchronize do
        @output = yield @input
        @waiter.wakeup if @waiter
        @output
      end
    end

    def value
      @mutex.synchronize do
        return @output if defined? @output
        @waiter = Thread.current
        @mutex.sleep
        @output
      end
    end
  end

  def initialize(host, port=80, use_ssl: port == 443, workers: 2)
    @queue = Queue.new
    workers.times {start_worker(host, port, use_ssl)}
  end

  def fetch(paths)
    paths.map do |item|
      request = WorkRequest.new(item)
      @queue << request
      request
    end.each {|req| yield req.value}
  end

  private

  def start_worker(host, port, use_ssl)
    Thread.new do
      client = Net::HTTP.new(host, port)
      client.use_ssl = use_ssl
      client.start
      loop {@queue.pop.process {|path| do_request(client, path)}}
    end.abort_on_exception = true
  end

  def do_request(client, path)
    response = client.get(path)
    response.value
    response.body
  rescue
    tries = tries.to_i + 1
    client.finish
    client.start
    retry if tries < 3
    raise
  end
end