起始url的调度原理

Posted 2020-12-10 xiongfanyong

# -*- coding: utf-8 -*-
import scrapy
from xdb.items import XdbItem
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http.cookies import CookieJar
from scrapy.http import Request
from urllib.parse import urlencode

class ChoutiSpider(scrapy.Spider):
    name = ‘chouti‘
    allowed_domains = [‘chouti.com‘]
    start_urls = [‘http://chouti.com/‘]
    # start_urls = [‘http://127.0.0.1:80/app01/login/‘]

    ‘‘‘
    scrapy引擎来爬虫中取起始URL：
        1. 调用start_request并获取返回值
        2. v = iter(返回值)
        3. req1 = 执行 v.__next__()
           req2 = 执行 v.__next__()
           req3 = 执行 v.__next__()
    ‘‘‘
    def start_requests(self):
        # 方式一：
        for url in self.start_urls:
            yield Request(url=url, callback=self.parse)
        # 方式二：
        req_list = []
        for url in self.start_urls:
            req_list.append(Request(url=url))
        return req_list