一个简单的爬虫case2

Posted 2021-01-24 bernieloveslife

目标是把这里的ppt什么的给下下来：https://web.stanford.edu/~jurafsky/slp3/

import requests
from bs4 import BeautifulSoup
import os
import socket
import time

socket.setdefaulttimeout(20)
url = ‘https://web.stanford.edu/~jurafsky/slp3/‘
r = requests.get(url)
root = ‘./crawl‘
soup = BeautifulSoup(r.text,"html.parser")
# print(soup)
count = 0
for link in soup.find_all("a"):
    name = link.get(‘href‘)
    if name.split(‘.‘)[-1] in [‘pdf‘,‘ppt‘,‘pptx‘]:
        print(url + name)
        path = root + ‘/‘+ name.split(‘/‘)[-1]
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):
            r = requests.get(url + name)
            with open(path, "wb") as f:
                f.write(r.content)
                f.close()
        count += 1
        print(count)
        r.close()
        time.sleep(1)
print("finished")