scrapy从命令行传值

Posted 2020-11-13 ptwg

1.新建文件run.py

from scrapy.cmdline import execute


# tmall：爬虫的名字
# pro=男装为需要传入的参数值
execute([‘scrapy‘, ‘crawl‘, ‘tmall‘, ‘-a‘, ‘pro=男装‘, ‘--nolog‘])

# 下面是无参数用法
# execute([‘scrapy‘, ‘crawl‘, ‘tmall‘, ‘--nolog‘])

 

2.爬虫.py中重写init方法，传入参数；（字典编码以字符串形式拼接到url后边）

# 倒入头文件 （字典编码后以参数形式拼接到url）
from urllib.parse import urlencode


class TmallSpider(scrapy.Spider):
    name = ‘tmall‘
    allowed_domains = [‘tmall.com‘]

    def __init__(self, pro=None, *args, **kwargs):
        super(TmallSpider, self).__init__(*args, **kwargs)
        self.params = {
            ‘q‘: pro,
            ‘total_Page‘: 1,
            ‘jumpto‘: 1,
        }

        self.start_url = ‘https://list.tmall.com/search_product.htm?‘ + urlencode(self.params)

    def start_requests(self):
        print(‘self.start_url:‘ + self.start_url)
        # yield scrapy.Request(
        #     url=self.start_url,
        #     callback=self.get_total_page,
        #     dont_filter=True,
        # )

    def get_total_page(self, response):
        pass