python爬虫：抓取新浪新闻内容（从当前时间到之前某个时间段），并用jieba分词，用于训练自己的分词模型

Posted 2020-09-22 古怪的一阵风

新浪新闻内容采用的是ajax动态显示内容，通过抓包，发现如下规律：

每次请求下一页，js那一栏都会出现新的url：

"http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1"
     "||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&"
     "format=json&page=1&callback=newsloadercallback"

"http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1"
     "||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&"
     "format=json&page=3&callback=newsloadercallback"

以上两个不同的是：page=？

Python实现代码：

 1 #-*- coding:utf-8 -*-
 2 __author__ = ‘Administrator‘
 3 
 4 import re
 5 from bs4 import BeautifulSoup
 6 import urllib.request
 7 import jieba
 8 import string
 9 import urllib.parse
10 from urllib.error import  HTTPError,URLError
11 import json
12 
13 def get_page(num):
14    return ("http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1"
15      "||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&"
16      "format=json&page={}&callback=newsloadercallback").format(str(num))
17 
18 def get_url(page_url):
19     #解决请求路径中含义中文或特殊字符
20     page_url=urllib.parse.quote(page_url, safe=string.printable)
21     #print(page_url)
22     url_list=[]
23     try:
24         res = urllib.request.urlopen(page_url)
25     except HTTPError as e:
26         print(‘The server couldn\‘t fulfill the request.‘)
27         print(‘Error code: ‘, e.code)
28         return url_list
29     except URLError as e:
30         print(‘We failed to reach a server.‘)
31         print(‘Reason: ‘, e.reason)
32         return url_list
33     else:
34         if res.getcode()==200:
35             jsdata=res.read().decode("utf-8")
36             ‘‘‘
37             截取url方法一
38             ‘‘‘
39             # result=re.findall(‘"url":"http.*?\.s?html"‘,jsdata)#.*后面再加？就可以变成非贪婪模式
40             # for url in result:
41             #     url=url.split(":",maxsplit=1)[1]
42             #     url=url.replace(‘\\‘,"")
43             #     url_list.append(url)
44             ‘‘‘
45             截取url方法二
46             ‘‘‘
47             data=jsdata[21:-2]
48             data=re.sub(‘\‘‘,‘\"‘,data)
49             data=re.sub(r"\\u","",data)
50             jsondata=json.loads(data)
51             for dat in jsondata["result"]["data"]:
52                 url_list.append(dat["url"])
53             return url_list
54 
55 
56 
57 def get_context(new_url):
58     #解决请求路径中含义中文或特殊字符
59     httpurl=urllib.parse.quote(new_url, safe=string.printable)
60     #print(httpurl)
61     #print(type(httpurl))
62     try:
63         html=urllib.request.urlopen(httpurl)
64     except HTTPError as e:
65         print(‘The server couldn\‘t fulfill the request.‘)
66         print(‘Error code: ‘, e.code)
67     except URLError as e:
68         print(‘We failed to reach a server.‘)
69         print(‘Reason: ‘, e.reason)
70     else:
71         if html.getcode()==200:
72             res=html.read().decode("utf-8")
73             #print(res)
74             soup=BeautifulSoup(res,‘html.parser‘)
75             #print(soup.prettify)
76             result={}
77             result["article"]=‘‘.join([p.text.strip() for p in soup.select(‘#artibody p‘)[:-1]])
78             context=result[‘article‘]
79             pattern=‘，|。|“|”|？|！|：|《|》|、|；|·|——| |‘|’|,|\?|\.|\!|`|~|\@|\#|\$|%|\^|\&|\*|（|）|\(|\)|-|\_|\+|=|\[|\]|\{|\}|"|\‘|\<|\>|\||‘
80             li=re.split(pattern,context)
81             #print("li")
82             with open(r".\traindata.txt",‘a‘,encoding=‘utf-8‘) as file:
83                 for l in li:
84                     if l!="":
85                         sentence = " ".join(jieba.cut(l))
86                         file.write(sentence + ‘\n‘)
87 
88 if __name__=="__main__":
89     for i in range(1,1001):
90         print("第 %d 页" % i)
91         page_url=get_page(i)
92         url_list=get_url(page_url)
93         #print(url_list) #[‘"http://news.sina.com.cn/c/nd/2017-06-11/doc-ifyfzhac1171724.shtml"‘, ...],双引号外层还有单引号
94         if url_list:
95             for url in url_list:
96                 #print(eval(url))
97                 #print(type(url))
98                 # get_context(eval(url))#针对方法一截取url
99                 get_context(url)#针对方法二截取url