爬取所有校园新闻

Posted yezless

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了爬取所有校园新闻相关的知识,希望对你有一定的参考价值。

1.获取单条新闻的#标题#链接#时间#来源#内容 #点击次数,并包装成一个函数。

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
url=http://news.gzcc.cn/html/xiaoyuanxinwen/
res=requests.get(url)
res.encoding=utf-8   
soup=BeautifulSoup(res.text,html.parser)
 
for news in soup.select(li):
    if len(news.select(.news-list-title))>0:
        title=news.select(.news-list-title)[0].text
        url=news.select(a)[0][href]
        time=news.select(.news-list-info)[0].contents[0].text
        dt=datetime.strptime(time,%Y-%m-%d)                           
        source=news.select(.news-list-info)[0].contents[1].text
        
        print(title,url,time,dt,source)

2.获取一个新闻列表页的所有新闻的上述详情,并包装成一个函数。

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
url=http://news.gzcc.cn/html/xiaoyuanxinwen/
res=requests.get(url)
res.encoding=utf-8   
soup=BeautifulSoup(res.text,html.parser)

for news in soup.select(li):
    if len(news.select(.news-list-title))>0:
        title=news.select(.news-list-title)[0].text
        url=news.select(a)[0][href]
        time=news.select(.news-list-info)[0].contents[0].text
        dt=datetime.strptime(time,%Y-%m-%d)                           
        source=news.select(.news-list-info)[0].contents[1].text
        
        resd=requests.get(url)
        resd.encoding=utf-8
        soupd=BeautifulSoup(resd.text,html.parser)
        ar=soupd.select(.show-content)[0].text
        print(title,url,time,dt,source,ar)
        break

3.获取所有新闻列表页的网址,调用上述函数。

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
url=http://news.gzcc.cn/html/xiaoyuanxinwen/
res=requests.get(url)
res.encoding=utf-8   
soup=BeautifulSoup(res.text,html.parser)

        


def getonepage(listurl):
    res=requests.get(listurl)
    res.encoding=utf-8   
    soup=BeautifulSoup(res.text,html.parser)

    
for news in soup.select(li):
    if len(news.select(.news-list-title))>0:
        title=news.select(.news-list-title)[0].text
        url=news.select(a)[0][href]
        time=news.select(.news-list-info)[0].contents[0].text
        dt=datetime.strptime(time,%Y-%m-%d)                           
        source=news.select(.news-list-info)[0].contents[1].text
        
        resd=requests.get(url)
        resd.encoding=utf-8
        soupd=BeautifulSoup(resd.text,html.parser)
        ar=soupd.select(.show-content)[0].text

        print(title,url,time,dt,source)

getonepage(http://news.gzcc.cn/html/xiaoyuanxinwen/index.html)
res.encoding=utf-8   
soup=BeautifulSoup(res.text,html.parser)
page= int(soup.select(.a1)[0].text.rstrip())//10+1
for i in range(2,page+1):
    listurl=http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html.format(i)
    getonepage(listurl)

 

4.完成所有校园新闻的爬取工作。

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
url=http://news.gzcc.cn/html/xiaoyuanxinwen/
res=requests.get(url)
res.encoding=utf-8   
soup=BeautifulSoup(res.text,html.parser)

        

def getclick(newurl):
    id = re.match(http://news.gzcc.cn/html/2017/xiaoyuanxinwen_(.*).html,newurl).groups()[0].split(/)[1]
    clickurl = http://oa.gzcc.cn/api.php?op=count&id=8301&modelid=80.format(id)
    click = int(requests.get(clickurl).text.split(.)[-1].lstrip("html(‘").rstrip("‘);"))
    return(click)

def getonepage(listurl):
    res=requests.get(listurl)
    res.encoding=utf-8   
    soup=BeautifulSoup(res.text,html.parser)

    
for news in soup.select(li):
    if len(news.select(.news-list-title))>0:
        title=news.select(.news-list-title)[0].text
        url=news.select(a)[0][href]
        time=news.select(.news-list-info)[0].contents[0].text
        dt=datetime.strptime(time,%Y-%m-%d)                           
        source=news.select(.news-list-info)[0].contents[1].text
        
        resd=requests.get(url)
        resd.encoding=utf-8
        soupd=BeautifulSoup(resd.text,html.parser)
        ar=soupd.select(.show-content)[0].text

        click=getclick(url)
        print(title,url,time,dt,source,click)

getonepage(http://news.gzcc.cn/html/xiaoyuanxinwen/index.html)
res.encoding=utf-8   
soup=BeautifulSoup(res.text,html.parser)
page= int(soup.select(.a1)[0].text.rstrip())//10+1
for i in range(2,4):
    listurl=http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html.format(i)
    getonepage(listurl)

 

以上是关于爬取所有校园新闻的主要内容,如果未能解决你的问题,请参考以下文章

爬取所有校园新闻

爬取所有校园新闻

爬取所有校园新闻

爬取所有校园新闻

爬取所有校园新闻

爬取所有校园新闻