Python爬虫爬企查查数据
Posted 68xi
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Python爬虫爬企查查数据相关的知识，希望对你有一定的参考价值。
因为制作B2b网站需要，需要入库企业信息数据。所以目光锁定企查查数据，废话不多说，开干！
#-*- coding-8 -*- import requests import lxml import sys from bs4 import BeautifulSoup import xlwt import time import urllib def craw(url,key_word,x): User_Agent = ‘Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0‘ # if x == 0: # re = ‘http://www.qichacha.com/search?key=‘+key_word # else: # re = ‘https://www.qichacha.com/search?key=#p:&‘.format(key_word,x-1) re = r‘https://www.qichacha.com/search?key=‘+key_word headers = ‘Host‘:‘www.qichacha.com‘, ‘Connection‘: ‘keep-alive‘, ‘Accept‘:r‘text/html, */*; q=0.01‘, ‘X-Requested-With‘: ‘XMLHttpRequest‘, ‘User-Agent‘:r‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36‘, ‘Referer‘: re, ‘Accept-Encoding‘:‘gzip, deflate, br‘, ‘Accept-Language‘:‘zh-CN,zh;q=0.9‘, ‘Cookie‘:r‘xxxxxxxxx这里换成你的cookiexxxxxxxx这里换成你的cookiexxxxxxxxx这里换成你的cookiexxxxxxx‘, try: response = requests.get(url,headers = headers) if response.status_code != 200: response.encoding = ‘utf-8‘ print(response.status_code) print(‘ERROR‘) soup = BeautifulSoup(response.text,‘lxml‘) except Exception: print(‘请求都不让，这企查查是想逆天吗？？？‘) try: com_all_info = soup.find_all(class_=‘m_srchList‘)[0].tbody com_all_info_array = com_all_info.select(‘tr‘) print(‘开始爬取数据，请勿打开excel‘) for i in range(0,len(com_all_info_array)): # try: temp_g_name = com_all_info_array[i].select(‘td‘)[2].select(‘.ma_h1‘)[0].text #获取公司名 temp_g_tag = com_all_info_array[i].select(‘td‘)[2].select(‘.search-tags‘)[0].text #获取公司标签 temp_r_name = com_all_info_array[i].select(‘td‘)[2].select(‘p‘)[0].a.text #获取法人名 temp_g_money = com_all_info_array[i].select(‘td‘)[2].select(‘p‘)[0].select(‘span‘)[0].text.strip(‘注册资本：‘) #获取注册资本 temp_g_date = com_all_info_array[i].select(‘td‘)[2].select(‘p‘)[0].select(‘span‘)[1].text.strip(‘成立日期：‘) #获取公司注册时间 temp_r_email = com_all_info_array[i].select(‘td‘)[2].select(‘p‘)[1].text.split(‘\n‘)[1].strip().strip(‘邮箱：‘) #获取法人Email temp_r_phone = com_all_info_array[i].select(‘td‘)[2].select(‘p‘)[1].select(‘.m-l‘)[0].text.strip(‘电话：‘) #获取法人手机号 temp_g_addr = com_all_info_array[i].select(‘td‘)[2].select(‘p‘)[2].text.strip().strip(‘地址：‘) #获取公司地址 temp_g_state = com_all_info_array[i].select(‘td‘)[3].select(‘.nstatus.text-success-lt.m-l-xs‘)[0].text.strip() #获取公司状态 g_name_list.append(temp_g_name) g_tag_list.append(temp_g_tag) r_name_list.append(temp_r_name) g_money_list.append(temp_g_money) g_date_list.append(temp_g_date) r_email_list.append(temp_r_email) r_phone_list.append(temp_r_phone) g_addr_list.append(temp_g_addr) g_state_list.append(temp_g_state) # except Exception: # print(‘错误！‘) except Exception: print(‘好像被拒绝访问了呢...请稍后再试叭...‘) if __name__ == ‘__main__‘: global g_name_list global g_tag_list global r_name_list global g_money_list global g_date_list global r_email_list global r_phone_list global g_addr_list global g_state_list g_name_list=[] g_tag_list=[] r_name_list=[] g_money_list=[] g_date_list=[] r_email_list=[] r_phone_list=[] g_addr_list=[] g_state_list=[] key_word = input(‘请输入您想搜索的关键词：‘) num = int(input(‘请输入您想检索的次数：‘))+1 sleep_time = int(input(‘请输入每次检索延时的秒数：‘)) key_word = urllib.parse.quote(key_word) print(‘正在搜索，请稍后‘) for x in range(1,num): url = r‘https://www.qichacha.com/search_index?key=&ajaxflag=1&p=&‘.format(key_word,x) s1 = craw(url,key_word,x) time.sleep(sleep_time) workbook = xlwt.Workbook() #创建sheet对象，新建sheet sheet1 = workbook.add_sheet(‘企查查数据‘, cell_overwrite_ok=True) #---设置excel样式--- #初始化样式 style = xlwt.XFStyle() #创建字体样式 font = xlwt.Font() font.name = ‘仿宋‘ # font.bold = True #加粗 #设置字体 style.font = font #使用样式写入数据 print(‘正在存储数据，请勿打开excel‘) #向sheet中写入数据 name_list = [‘公司名字‘,‘公司标签‘,‘法定法人‘,‘注册资本‘,‘成立日期‘,‘法人邮箱‘,‘法人电话‘,‘公司地址‘,‘公司状态‘] for cc in range(0,len(name_list)): sheet1.write(0,cc,name_list[cc],style) for i in range(0,len(g_name_list)): print(g_name_list[i]) sheet1.write(i+1,0,g_name_list[i],style)#公司名字 sheet1.write(i+1,1,g_tag_list[i],style)#公司标签 sheet1.write(i+1,2,r_name_list[i],style)#法定法人 sheet1.write(i+1,3,g_money_list[i],style)#注册资本 sheet1.write(i+1,4,g_date_list[i],style)#成立日期 sheet1.write(i+1,5,r_email_list[i],style)#法人邮箱 sheet1.write(i+1,6,r_phone_list[i],style)#法人电话 sheet1.write(i+1,7,g_addr_list[i],style)#公司地址 sheet1.write(i+1,8,g_state_list[i],style)#公司状态 #保存excel文件，有同名的直接覆盖 workbook.save(r"D:\wyy-qcc-"+time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) +".xls") print(‘保存完毕~‘)
以上是关于Python爬虫爬企查查数据的主要内容，如果未能解决你的问题，请参考以下文章