用Python写了个简单小爬虫

Posted 2020-08-11

 1 #coding: utf-8
 2 
 3 import requests,urllib2,urlparse,time,sys
 4 from bs4 import BeautifulSoup
 5 
 6 #获取百度url wd查询关键子 pn第几页  rn 每页多少条 最大50条
 7 #getbaidu.py 关键字 爬取页数 每页数量
 8 #getbaidu.py example 10 50
 9 #百度特性 每页显示条数为10 20 30类型的整数
10 
11 
12 type = sys.getfilesystemencoding()
13 
14 def geturl(wd,pn,rn):
15     #geturl(关键字，爬取页数，单页显示数量)
16 
17     #配置
18     target = ‘http://www.baidu.com/s‘
19     i = pn #记录页数
20     pn = pn*rn-1 #显示至当前页最后一个标题
21     url = target+‘?wd=‘+wd+‘&pn=‘+str(pn)+‘&rn=‘+str(rn)
22 
23     #获取页面主体并分析
24     body = getbody(url)
25     soup = BeautifulSoup(body,‘html.parser‘)
26 
27     #分析
28     tags = soup.find_all(‘a‘,attrs={‘class‘:‘c-showurl‘})
29     links = []
30     for tag in tags:
31         href = tag.attrs[‘href‘]
32         #打印跳转后的url
33         #print urlparse.urlparse(requests.head(href).headers[‘Location‘]).netloc
34         href = requests.head(href).headers[‘Location‘]
35         links.append(href)
36         print href
37 
38     #判断最终页
39     if int(i) == 1: 
40         print ‘\r--------     Web spider is end     -------\r‘
41 
42     else:
43         print ‘\n#############     Next page,sleep 1 sencound    ##################            ‘+str(i)+‘   \n‘
44         time.sleep(1)
45         geturl(wd,int(i)-1,rn)
46 
47 
48 def getbody(url):
49     request = urllib2.Request(url)
50     response = urllib2.urlopen(request)
51     return response.read()
52 
53 if __name__ == ‘__main__‘:
54     key = str(sys.argv[1])
55     page_num = int(sys.argv[2])
56     show_num = int(sys.argv[3])
57     geturl(key,page_num,show_num)

配合着sqlmap api一起用，批量检测sql注入应该会很棒~