ruby simpledesktops_scraper.rb

Posted 2021-05-16

require 'rubygems'
require 'hpricot'
require 'open-uri'

@url = "http://simpledesktops.com/browse/"

def scrape_page(url, page_number=0)
  puts "Scraping #{url}"
  Dir.mkdir "#{page_number}" unless File.exists? "#{page_number}"
  doc = Hpricot( open( url ) )

  ( doc/".desktop > a" ).each do |link|
    href = link.attributes["href"]
    path = "#{page_number}/#{File.basename href}"
    unless File.exists? path
      print "#{href}... "
      open( href ) do |image|
        File.open( path, "wb" ) do |f|
          f.write image.read
          puts "Saved #{path}"
        end
      end
    else
      puts "#{path} exists"
    end
  end

  next_page = ( doc/".pagination .older" )
  unless next_page.length == 0
    u = URI.parse url
    next_page_url = URI::HTTP.build({ :host => u.host, :path => next_page[0].attributes["href"] }).to_s
    puts "Next page: #{next_page_url}"
    page_number += 1
    scrape_page next_page_url, page_number
  end
end

scrape_page @url