Python 爬虫

Posted 2020-12-30 暗香丶

 # 头条街拍图片爬取


 1 import re
 2 import requests
 3 from urllib import request
 4 import json
 5 import os
 6 i = 0
 7 headers = {
 8     ‘user-agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36‘
 9 }
10 while True:
11     pag_all_url = ‘https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab‘
12     i += 20
13     full_pag_url = pag_all_url.format(i)
14     # print(full_pag_url) # 街拍的首页 根据不同的i来请求ajax,从而获得所有的街拍网址 像这样https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D
15     pag_html = requests.get(full_pag_url,headers = headers).text
16     pag_html_str = str(json.loads(pag_html))
17     # print(pag_html_str) #把网页转化为字符串 进行正则匹配
18     img_pag_id = re.findall(r‘‘item_source_url‘: ‘/group/(d*)/‘,‘,pag_html_str)
19     # print(img_pag_id) #获得每个街拍的url like this--->https://www.toutiao.com/a6590127156037157379/
20     for l in img_pag_id:    #图片下载
21         img_all_url = ‘https://www.toutiao.com/a{}‘
22         full_url = img_all_url.format(l)
23         # print(full_url)#图片的url  print(full_pag_url)#图片所在的url
24         html = requests.get(full_url,headers=headers).text
25         pattern = r‘gallery: JSON.parse((.*)),‘
26         ans1 = re.search(pattern,html)
27         try:
28             ans1_str = json.loads(ans1[1])
29             ans1_dic = json.loads(ans1_str)
30             # print(ans1_dic)
31             # if not os.path.exists(‘1‘):
32             #     os.mkdir(‘1‘)
33             for q in ans1_dic[‘sub_images‘]:
34                 img_url = q[‘url‘]
35                 print(img_url)
36                 filename = ‘1/‘ + img_url.split(‘/‘)[-1] + ‘.jpg‘
37                 request.urlretrieve(img_url, filename)
38         except:continue