爬取新闻列表

Posted 2020-10-10 03郭丽红

import requests
import re
from bs4 import BeautifulSoup
url=\'http://news.gzcc.cn/html/xiaoyuanxinwen/\'
res=requests.get(url)
res.encoding=\'utf-8\'
soup=BeautifulSoup(res.text,\'html.parser\')

n=int(soup.select(\'.a1\')[0].text.rstrip(\'条\'))
pages=n//10+1
for i in range(2,pages+1):
    pagesurl=\'http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html\'.format(i)
    print(pagesurl)

import requests
import re
from bs4 import BeautifulSoup
def getclick(newurl):    #函数：采用匹配方式，动态获取每一文章点击次数
    id=re.search(\'_(.*).html\',newurl).group(1).split(\'/\')[1]
    clickurl=\'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80\'.format(id)
    click=int(requests.get(\'http://oa.gzcc.cn/api.php?op=count&id=4662&modelid=80\').text.split(\'.\')[-1].lstrip("html(\'").rstrip("\');"))
    return click

def getonepages(eveylisturl):  #函数：每一页的每一文章
    res=requests.get(eveylisturl)
    res.encoding=\'utf-8\'
    soup=BeautifulSoup(res.text,\'html.parser\')

    for news in soup.select(\'li\'):
        if len(news.select(\'.news-list-title\'))>0:
            title=news.select(\'.news-list-title\')[0].text #标题
            url=news.select(\'a\')[0][\'href\']#网址
            time=news.select(\'.news-list-info\')[0].contents[0].text#时间
            bumen=news.select(\'.news-list-info\')[0].contents[1].text#院系
            #txt=news.select(\'.news-list-description\')[0].text #正文

            resd=requests.get(url)
            resd.encoding=\'utf-8\'
            soupd=BeautifulSoup(resd.text,\'html.parser\')
            detail=soupd.select(\'.show-content\')[0].text #获取正文
            count=getclick(url)      #调用函数获得文章的点击次数
  
            print(title,count)

gzcc=\'http://news.gzcc.cn/html/xiaoyuanxinwen/\'  #函数开头
res=requests.get(gzcc)
res.encoding=\'utf-8\'
soup=BeautifulSoup(res.text,\'html.parser\')

        
     
n=int(soup.select(\'.a1\')[0].text.rstrip(\'条\'))
pages=n//10+1                    #计算多少条多少页
for i in range(2,5):
    pagesurl=\'http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html\'.format(i)
    getonepages(pagesurl)