scrapy-splash抓取动态数据例子二
Posted shaomine
tags:
篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了scrapy-splash抓取动态数据例子二相关的知识,希望对你有一定的参考价值。
一、介绍
本例子用scrapy-splash抓取一点资讯网站给定关键字抓取咨询信息。
给定关键字:打通;融合;电视
抓取信息内如下:
1、资讯标题
2、资讯链接
3、资讯时间
4、资讯来源
二、网站信息
三、数据抓取
针对上面的网站信息,来进行抓取
1、首先抓取信息列表,由于信息列表的class值有“item doc style-small-image style-content-middle” 和“item doc style-multi-image”两种情况,所以用contains包含item doc style-的语法来抓
抓取代码:sels = site.xpath(\'//a[contains(@class,"item doc style-")]\')
2、抓取标题
抓取代码:titles = sel.xpath(\'.//div[@class="doc-title"]/text()\')
3、抓取链接
抓取代码:it[\'url\'] = \'http://www.yidianzixun.com\' + sel.xpath(\'.//@href\')[0].extract()
4、抓取日期
抓取代码:dates = sel.xpath(\'.//span[@class="date"]/text()\')
5、抓取来源
抓取代码:it[\'source\'] = sel.xpath(\'.//span[@class="source"]/text()\')[0].extract()
四、完整代码
1、yidianzixunSpider
# -*- coding: utf-8 -*- import scrapy from scrapy import Request from scrapy.spiders import Spider from scrapy_splash import SplashRequest from scrapy_splash import SplashMiddleware from scrapy.http import Request, htmlResponse from scrapy.selector import Selector from scrapy_splash import SplashRequest from splash_test.items import SplashTestItem import IniFile import sys import os import re import time reload(sys) sys.setdefaultencoding(\'utf-8\') # sys.stdout = open(\'output.txt\', \'w\') class yidianzixunSpider(Spider): name = \'yidianzixun\' start_urls = [ \'http://www.yidianzixun.com/channel/w/打通?searchword=打通\', \'http://www.yidianzixun.com/channel/w/融合?searchword=融合\', \'http://www.yidianzixun.com/channel/w/电视?searchword=电视\' ] # request需要封装成SplashRequest def start_requests(self): for url in self.start_urls: print url index = url.rfind(\'=\') yield SplashRequest(url , self.parse , args={\'wait\': \'2\'}, meta={\'keyword\': url[index + 1:]} ) def date_isValid(self, strDateText): \'\'\' 判断日期时间字符串是否合法:如果给定时间大于当前时间是合法,或者说当前时间给定的范围内 :param strDateText: 四种格式 \'2小时前\'; \'2天前\' ; \'昨天\' ;\'2017.2.12 \' :return: True:合法;False:不合法 \'\'\' currentDate = time.strftime(\'%Y-%m-%d\') if strDateText.find(\'分钟前\') > 0 or strDateText.find(\'刚刚\') > -1: return True, currentDate elif strDateText.find(\'小时前\') > 0: datePattern = re.compile(r\'\\d{1,2}\') ch = int(time.strftime(\'%H\')) # 当前小时数 strDate = re.findall(datePattern, strDateText) if len(strDate) == 1: if int(strDate[0]) <= ch: # 只有小于当前小时数,才认为是今天 return True, currentDate return False, \'\' def parse(self, response): site = Selector(response) keyword = response.meta[\'keyword\'] it_list = [] sels = site.xpath(\'//a[contains(@class,"item doc style-")]\') for sel in sels: dates = sel.xpath(\'.//span[@class="date"]/text()\') if len(dates) > 0: flag, date = self.date_isValid(dates[0].extract()) if flag: titles = sel.xpath(\'.//div[@class="doc-title"]/text()\') if len(titles) > 0: title = str(titles[0].extract()) if title.find(keyword) > -1: it = SplashTestItem() it[\'title\'] = title it[\'url\'] = \'http://www.yidianzixun.com\' + sel.xpath(\'.//@href\')[0].extract() it[\'date\'] = date it[\'keyword\'] = keyword it[\'source\'] = sel.xpath(\'.//span[@class="source"]/text()\')[0].extract() it_list.append(it) if len(it_list) > 0: return it_list
2、SplashTestItem
import scrapy class SplashTestItem(scrapy.Item): #标题 title = scrapy.Field() #日期 date = scrapy.Field() #链接 url = scrapy.Field() #关键字 keyword = scrapy.Field() #来源网站 source = scrapy.Field()
3、SplashTestPipeline
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don\'t forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import codecs import json class SplashTestPipeline(object): def __init__(self): # self.file = open(\'data.json\', \'wb\') self.file = codecs.open( \'spider.txt\', \'w\', encoding=\'utf-8\') # self.file = codecs.open( # \'spider.json\', \'w\', encoding=\'utf-8\') def process_item(self, item, spider): line = json.dumps(dict(item), ensure_ascii=False) + "\\n" self.file.write(line) return item def spider_closed(self, spider): self.file.close()
4、settings.py
# -*- coding: utf-8 -*- # Scrapy settings for splash_test project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html ITEM_PIPELINES = { \'splash_test.pipelines.SplashTestPipeline\':300 } BOT_NAME = \'splash_test\' SPIDER_MODULES = [\'splash_test.spiders\'] NEWSPIDER_MODULE = \'splash_test.spiders\' SPLASH_URL = \'http://192.168.99.100:8050\' # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = \'splash_test (+http://www.yourdomain.com)\' # Obey robots.txt rules ROBOTSTXT_OBEY = False DOWNLOADER_MIDDLEWARES = { \'scrapy_splash.SplashCookiesMiddleware\': 723, \'scrapy_splash.SplashMiddleware\': 725, \'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware\': 810, } SPIDER_MIDDLEWARES = { \'scrapy_splash.SplashDeduplicateArgsMiddleware\': 100, } DUPEFILTER_CLASS = \'scrapy_splash.SplashAwareDupeFilter\' HTTPCACHE_STORAGE = \'scrapy_splash.SplashAwareFSCacheStorage\' # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) #COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # \'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\', # \'Accept-Language\': \'en\', #} # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # \'splash_test.middlewares.SplashTestSpiderMiddleware\': 543, #} # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # \'splash_test.middlewares.MyCustomDownloaderMiddleware\': 543, #} # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { # \'scrapy.extensions.telnet.TelnetConsole\': None, #} # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # \'splash_test.pipelines.SplashTestPipeline\': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = \'httpcache\' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = \'scrapy.extensions.httpcache.FilesystemCacheStorage\'
以上是关于scrapy-splash抓取动态数据例子二的主要内容,如果未能解决你的问题,请参考以下文章