python爬取中国知网部分论文信息

Posted 2021-12-02 ljy1227476113

爬取指定主题的论文，并以相关度排序。

 1 #!/usr/bin/python3
 2 # -*- coding: utf-8 -*- 
 3 import requests
 4 import linecache
 5 import random
 6 from bs4 import BeautifulSoup
 7 
 8 if __name__=="__main__":
 9     keywords=‘通信‘ ### 查询的主题 
10     n=0
11     target=‘http://search.cnki.net/search.aspx?q=‘+str(keywords)+‘&rank=relevant&cluster=all&val=CJFDTOTAL&p={}‘
12     user_agent = ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/54.0.2840.99 Safari/537.36‘
13     headers = {‘User-Agent‘:user_agent}
14     for i in range(10):
15         i=i*15
16         target=target.format(i)
17         req=requests.get(url=target)
18         html=req.text
19         html=html.replace(‘<br>‘,‘ ‘).replace(‘<br/>‘,‘ ‘).replace(‘/>‘,‘>‘)
20         bf=BeautifulSoup(html,"html.parser")
21         texts=bf.find(‘div‘,class_=‘articles‘)
22         texts_div=texts.find_all(‘div‘,class_=‘wz_content‘)
23         for item in texts_div:
24             item_name=item.find(‘a‘).text
25             item_href=item.find(‘a‘)[‘href‘]
26             item_refer2=item.find(‘span‘,class_=‘count‘).text
27             print(‘{} {} {}\n‘.format(item_name,item_href,item_refer2))
28     print(n)