scrapy模拟登陆小例子

Posted 2020-10-22 lilied

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request,FormRequest


class IqianSpider(scrapy.Spider):
    name = ‘iqian‘
    allowed_domains = [‘edu.iqianxxx.com‘]
    # start_urls = [‘http://edu.iqianxxx.com/‘]
    headers = {‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER‘}
    
    # 入口
    def start_requests(self):
        start_url = "http://edu.iqianxxx.com/index_user_login.html"
        return [Request(url=start_url, meta={"cookiejar": 1}, callback=self.login)]

    def login(self, response):
        data = {
            "number": "xxxx",
            "passwd": "xxxx",
        }
        print("登录中...............")
        return [FormRequest.from_response(response,
                                        meta={"cookiejar": response.meta["cookiejar"]},
                                        headers=self.headers,
                                        formdata=data,
                                        callback=self.next1,
        )]
    
    #需要在跳转一次，以防中间页面
    def next1(self, response):
        yield Request(url="http://edu.iqianxxx.com/index_user_index.html", callback=self.next2)

    def next2(self, response):
        print(response.url)
        pass