sys设置递归深度

Posted 2021-03-09 fran-py-

发现了一个练习爬虫的网站GlidedSky

过完了第一关，第二关需要爬取1000个网页，爬取到第956个网页时报错：

RecursionError: maximum recursion depth exceeded while calling a Python object

 解决方法：

import sys
sys.setrecursionlimit(100000)

https://blog.csdn.net/Ren_ger/article/details/85068955

 

附上我爬取第二关的代码：

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from lxml import etree
import sys

sys.setrecursionlimit(100000)
url=‘http://glidedsky.com/level/web/crawler-basic-2?page=1‘
browser=webdriver.Chrome()
wait=WebDriverWait(browser,10)
browser.get(url)

def index_from_page(page):
    print(‘正在爬取第‘,page,‘页‘)
    try:
        if page<1000:
            wait.until(EC.presence_of_element_located((By.XPATH,‘//*[@id="app"]/main/div[1]/div/div‘)))
            button=wait.until(EC.element_to_be_clickable((By.XPATH,‘//*[@id="app"]/main/div[1]/ul/li//a[@rel="next"] ‘)))
            get_sum(page)
            time.sleep(1)
            button.click()
            index_from_page(page+1)
        elif page==1000:
            get_sum(page)
    except TimeoutException:
            index_from_page(page)


n=0
def get_sum(url):
    global n
    html=browser.page_source
    html=etree.HTML(html)
    item=html.xpath(‘/html/body/div/main/div[1]/div/div/div/div/text()‘)
    for num in item:
        num=eval(get_number(num))
        n+=num
def get_number(num):
    for i in range(2):
        num=num.strip(‘ ‘)
        num=num.strip(‘
‘)
    return num                      


a=time.perf_counter()
index_from_page(1)
b=time.perf_counter()
print(‘time:‘,b-a)