ruby 我可恶吗？

Posted 2021-05-15

require 'crawlers/yellow_pages_crawler' 

class YellowPages

  DEFAULT_PAGE_LENGTH = 100
  DEFAULT_CONCURRENT_THREADS = 10

  def initialize(api_key, sandbox_enabled)
    @client = YellowApi.new(:apikey => api_key, :sandbox_enabled => sandbox_enabled)
  end

  def trigger(what, where)
    get_listings(what, where) do |listings|
      urls = {}
      listings.each do |listing|
        unless listing.address.prov.empty? or listing.address.city.empty?
          business_identifier = [listing.address.prov, listing.address.city, listing.name, listing.id.to_i]
          urls[generate_business_url(*business_identifier)] = business_identifier
        end
      end
      YellowPagesCrawler.new(:concurrency => DEFAULT_CONCURRENT_THREADS, :locations => urls).start
    end
  end

  private
  def get_listings(what, where, current_page=1, max_pages=50)
    while current_page < max_pages
      wait 0.5 do
        result = @client.find_business(what, where, { :pg => current_page, :pgLen => DEFAULT_PAGE_LENGTH })
      
        yield result.listings
      
        current_page += 1
        page_count = result.summary.pageCount
        max_pages = page_count if page_count < max_pages
      end
    end
  end

  def generate_business_url(province, city, business_name, id)
    s_province = @client.expand_province(province)
    s_city = @client.normalize(city)
    s_business_name = @client.normalize(business_name.strip)

    "http://www.yellowpages.ca/bus/#{s_province}/#{s_city}/#{s_business_name}/#{id}.html"
  end

  def wait(t, &block)
    sleep t
    yield
  end
end