第一阶段冲刺4

Posted 2020-12-10 shumouren

今天终于实现了爬虫，爬取到了一定的信息

 

代码：

 

# -*- coding:utf-8 -*-

import requests

from bs4 import BeautifulSoup

import bs4

from selenium import webdriver

from time import sleep

import time

from PIL import Image

 

 

 

class Login(object):

 

       def __init__(self):

             

             

              options=webdriver.ChromeOptions()

              options.add_argument(‘--headless‘)

              options.add_argument(‘--disable-gpu‘)

              options.add_argument(‘--ignore-certificate-errors‘)

              options.add_argument("--disable-gpu")

              self.driver=webdriver.Chrome(options=options)

              self.driver.maximize_window()

              self.driver.set_window_size(‘1920‘,‘1080‘) #设置浏览器宽480，高800　

              self.driver.get(‘http://tiedao.vatuu.com/service/login.html?returnUrl=return‘)

              cookie = self.driver.get_cookies()

              print(type(cookie))

              print(cookie[0][‘value‘])

              self.cookie = cookie[0][‘value‘]

              print(type(self.cookie))

              self.headers = {

                            ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8‘,

                            ‘Accept-Encoding‘: ‘gzip, deflate‘,

                            ‘Accept-Language‘: ‘zh-CN,zh;q=0.9,en;q=0.8‘,

                            ‘Connection‘: ‘keep-alive‘,

                            ‘Cookie‘: self.cookie,

                            ‘Host‘: ‘tiedao.vatuu.com‘,

                            ‘Upgrade-Insecure-Requests‘: ‘1‘,

                            ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36

                                   (KHTML, like Gecko)Chrome/72.0.3626.121 Safari/537.36‘

                     }

              print(self.headers)

              self.url = ‘http://tiedao.vatuu.com/vatuu/StudentScoreInfoAction?setAction=studentScoreQuery&viewType=

                     studentScore&orderType=submitDate&orderValue=desc‘

             

       # 时间格式进行格式化

       def time_format(self):

              current_time = time.strftime(‘%Y%m%d%H%M%S‘, time.localtime(time.time()))

              return current_time

       def cut(self):

             

              # 截取全屏

              self.driver.save_screenshot("C:\Users\kang\Desktop\pachong\full.png")

              # 要截屏的目标元素

              element = self.driver.find_element_by_id("randomPhoto")

              print(element.location)

              print(element.size)

              # 获取element的顶点坐标

              xPiont = element.location[‘x‘]

              yPiont = element.location[‘y‘]

              # 获取element的宽、高

              element_width = xPiont + element.size[‘width‘]

              element_height = yPiont + element.size[‘height‘]

        

              picture = Image.open(‘C:\Users\kang\Desktop\pachong\full.png‘)

        

              ‘‘‘

              crop()--  一个显式的参数：一个4元组

            Image.crop(box=None):图像返回一个矩形区域,box是一个四元组 限定所述左，上，右，和下像素坐标

              参数：box--裁剪矩形，作为（左，上，右，下）-tuple;返回类型：Image；返回：一个Image对象

              所以你应该重写它：

              img.crop((414,122,650,338))

              #        ^    4-tuple    ^

              ‘‘‘

              picture = picture.crop((xPiont, yPiont, element_width-30, element_height))

              src = "C:\Users\kang\Desktop\pachong\"+self.time_format()+".png"

              picture.save(src)

             

      

      

             

       def login(self):

              self.cut()

              user = self.driver.find_element_by_id(‘username‘)

              passwor = self.driver.find_element_by_id(‘password‘)

              ranstring = self.driver.find_element_by_id(‘ranstring‘)

              confirm = self.driver.find_element_by_id(‘submit2‘)

              name = input("请输入:username")

              password = input("请输入:password")

              ran = input("请输入:验证码")

              user.send_keys(name)

              passwor.send_keys(password)

              ranstring.send_keys(ran)

              confirm.click()

             

             

def get_soup(url,cookies=‘‘):

       if cookies:

              html = get_html(url,cookies)

              soup = BeautifulSoup(html, ‘lxml‘)

              return soup

       else:

              html = get_html(url,‘‘)

              soup = BeautifulSoup(html, ‘lxml‘)

             

              return soup  

             

def html_utf(content):

       html=content

       html_doc=str(html,‘utf-8‘) #html_doc=html.decode("utf-8","ignore")

       return html_doc   

def get_html(url,cookies):

       try:

              r = requests.get(url, timeout=5, cookies=cookies)

              r.raise_for_status()

              return html_utf(r.content)

       except:

              return "ERROR"   

def score(url,cookie):

       # print(get_html(url,{‘JSESSIONID‘:cookie}))

       soup = get_soup(url,{‘JSESSIONID‘:cookie})

       table = soup.find(‘table‘, attrs={‘id‘:‘table3‘})

       print(table)

       trs = table.find_all(‘tr‘)

       contents = []

       for tr in trs:

              ths = tr.find_all(‘th‘)

              if ths:

                     for th in ths:

                            print(th.string.strip(),‘ | ‘,end=‘‘)

                     print(‘‘)

              else:

                     content = []

                     tds = tr.find_all(‘td‘)

                     for td in tds:

                            if td.string is None:

                                   pass

                            else:

                                   str = td.string.strip()

                                   content.append(str)

                                   print(str,‘ | ‘,end=‘‘)

                     contents.append(content)

                     print(‘‘)

       for con in contents:

              for c in con:

                     print(c,‘||‘,end=‘‘)

              print(‘‘)  

 

def course(url,cookie):

       soup = get_soup(url,{‘JSESSIONID‘:cookie})

       table = soup.find(‘table‘,attrs={‘class‘:‘table_border‘})

       trs = table.find_all(‘tr‘)

       contents = []

       for tr in trs:

              tds = tr.find_all(‘td‘)

              for td in tds:

                     res = td.find_all(text=True)

                     strs = ‘‘

                     for s in res:

                            strs+=s

                     # print(strs)

                     contents.append(strs)

                     print(‘+++++++++++++++++++++++++++++‘)

       ‘‘‘

       classList 按时间顺序排好的课程

       ‘‘‘

       classList = []                

       for i in range(1,8):

              for j in range(1,13):

                     content = ‘星期‘+str(i)+‘：第‘+str(j)+‘节：‘+contents[j*8+i]

                     print(content)

                     classList.append(content)

      

                    

def room(url,cookie):

       ‘‘‘

       首先构建表单

       ‘‘‘

       week = input(‘请输入周数：‘)

       day_num = input(‘请输入星期几：‘)

       class_num = input(‘请输入：

       1：第一二节

       2：第三四节

       3：整个上午

       4：第六七节

       5：第八九节

       6：整个下午

       7：晚自习‘)

       week = pow(2,int(week)-1)

 

       day_time = ‘‘

      

       if class_num == ‘1‘:

              day_time=‘0000000000011‘

       elif class_num == ‘2‘:

              day_time=‘0000000001100‘

       elif class_num == ‘3‘:

              day_time=‘0000000001111‘

       elif class_num == ‘4‘:

              day_time=‘0000001100000‘

       elif class_num == ‘5‘:

              day_time=‘0000110000000‘

       elif class_num == ‘6‘:

              day_time=‘0000111100000‘

       elif class_num == ‘7‘:

              day_time=‘0110000000000‘

       param = {‘setAction‘: ‘classroomQuery‘,

                     ‘PageAction‘: ‘Query‘,

                     ‘day_time_text‘: day_time,

                     ‘school_area_code‘: ‘1‘,

                     ‘building‘: ‘‘,

                     ‘week_no‘: str(week),

                     ‘day_no‘: str(int(day_num)),

                     ‘B1‘: ‘查询‘

              }

       cookie_copy = ‘JSESSIONID=‘+cookie

       print(param)

      

       headers = {

                            ‘Connection‘: ‘keep-alive‘,

                            ‘Cache-Control‘: ‘max-age=0‘,

                            ‘Origin‘: ‘http://tiedao.vatuu.com‘,

                            ‘Upgrade-Insecure-Requests‘: ‘1‘,

                            ‘Content-Type‘: ‘application/x-www-form-urlencoded‘,

                            ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36

                                   (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36‘,

                            ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,

                                   image/apng,*/*;q=0.8‘,

                            ‘Referer‘: ‘http://tiedao.vatuu.com/vatuu/CourseAction‘,

                            ‘Accept-Encoding‘: ‘gzip, deflate‘,

                            ‘Accept-Language‘: ‘zh-CN,zh;q=0.9,en;q=0.8‘,

                            ‘Cookie‘:  cookie_copy

                     }

       r = requests.post(url,data=param,headers=headers)

       soup = BeautifulSoup(html_utf(r.content), ‘lxml‘)

       table = soup.find(‘table‘,attrs={‘class‘:‘table_gray‘})

       trs = table.find_all(‘tr‘)

       contents = []

       for tr in trs:

              ths = tr.find_all(‘th‘)

              if ths:

                     for th in ths:

                            print(th.string.strip(),‘ | ‘,end=‘‘)

                     print(‘‘)

              else:

                     content = []

                     tds = tr.find_all(‘td‘)

                     for td in tds:

                            res = td.find_all(text=True)

                            strs = ‘‘

                            for s in res:

                                   strs+=s

                            strs = strs.strip()

                            print(strs,‘ || ‘,end=‘‘)

                            contents.append(strs)

              print(‘+++++++++++++++++++++++++++++‘)

      

                    

                           

      

if __name__ == ‘__main__‘:

       login = Login()

       # login.login()

       time.sleep(2)

       course_url = ‘http://tiedao.vatuu.com/vatuu/CourseAction?setAction=userCourseScheduleTable

              &viewType=studentQueryCourseList&selectTableType=ThisTerm&queryType=student‘

       room_url = ‘http://tiedao.vatuu.com/vatuu/CourseAction‘

       # score(login.url,login.cookie)

       print(login.cookie)

       cookie=‘8CD73FB791382490DA6F32187893B80E‘

       course(course_url,cookie)

       for i in range(1,100):

              room(room_url,cookie)

 

 

下一步准备进行遍历这些数据，展示空教室信息。