Python开发网络爬虫抓取某同城房价信息

Posted 2020-09-08

前言：

苦逼的我从某某城市换到另一个稍微大点的某某城市，面临的第一个问题就是买房，奋斗10多年，又回到起点，废话就不多说了，看看如何设计程序把某同城上的房价数据抓取过来。

方案：方案思路很简单，先把网页内容获取下来，通过一定规则对内容解析，保存成想要的格式

难点是对网页的解析，是一个比较细致的活，必须边输出，边调试。

具体实现：

获取网页内容：

def get_page(url):
    headers = {
        ‘User-Agent‘: r‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ‘
                      r‘Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3‘,
        ‘Referer‘: r‘http://jn.58.com/ershoufang/‘,
        ‘Host‘: r‘jn.58.com‘,
        ‘Connection‘: ‘keep-alive‘
    }
    timeout = 60
    socket.setdefaulttimeout(timeout)  # 设置超时
    req = request.Request(url, headers=headers)
    response = request.urlopen(req).read()
    page = response.decode(‘utf-8‘,‘ignore‘)
    return page

第二步解析网页：解析时要注意无效内容的处理，不然跑起来会报错，调试很麻烦

def get_58_house(url):   
    html = get_page(url)
    soup =  BeautifulSoup(html,"lxml")
    table =soup.find(id="main")
    df = pd.DataFrame(columns=["op_time","web","house_name","xq","xq1","price","per_price","room","m2","href","ts"])
    for tr in table.find_all(‘tr‘):
        try:
            str_name = tr.find("p","bthead").find("a","t").string.strip()
            str_link = tr.find("p","bthead").find("a","t")["href"]

           
            ##房产小区位置
            str_xq = list()  
            str_xq1= ‘‘
            str_xq2= ‘‘
            try:
                for s in tr.find_all("a","a_xq1")    :
                    str_xq.append(s.string.strip()) 
                str_xq1= str_xq[0]
                str_xq2= str_xq[1]
            except:
                pass
            ##房产特色
            str_ts =list()
            try:
                for s in tr.find("div","qj-listleft").stripped_strings:
                    str_ts.append(s)
            except:
                pass            

            ## 价格信息####################
            str_price =list()
            str_toal =‘‘
            str_per =‘‘
            str_room =‘‘
            str_m2 =‘‘
            try:
                for s in tr.find("div","qj-listright btall").stripped_strings:
                    str_price.append(s)
                str_toal = str_price[0]
                str_per  = re.findall(r"(\\d+\\.*\\d+)",str_price[1])
                str_room = str_price[2]
                str_m2  = re.findall(r"(\\d+\\.*\\d+)",str_price[3])           
            except:
                pass
        except Exception as e:
            print(‘Exception‘,":",e)
                       
        try: 
            row = {‘web‘:‘58同城‘,‘house_name‘:str_name,‘xq‘:str_xq1,‘xq1‘:str_xq2,‘price‘:str_toal,‘per_price‘:str_per,‘room‘:str_room,‘m2‘:str_m2,‘ts‘:‘‘.join(str_ts),‘href‘:str_link}
            newrow = pd.DataFrame(data=row,index=["0"])
            df=df.append(newrow,ignore_index=True)
        except Exception as e:
            print(‘Exception‘,":",e)
            f=open("log.txt",‘a‘)
            traceback.print_exc(file=f) 
            f.write(row) 
            f.flush() 
            f.close()
    df["op_time"]=time.strftime(‘%Y-%m-%d‘,time.localtime(time.time()))
    return df

第三步循环处理每页数据并保存数据：

def get_58_house_all():
    ##建立数据库连接
    engine = create_engine(‘oracle+cx_oracle://user:[email protected]/orcl‘)
    cnx = engine.connect() 
    ##先清除今天的数据
    ‘‘‘
    strSql = ‘delete from house where op_time=\\‘{}\\‘ ‘.format(time.strftime(‘%Y-%m-%d‘,time.localtime(time.time())))
    cnx.execute(strSql)
    ‘‘‘
    ##获取首页房产数据   
    str_http = "http://jn.58.com/ershoufang/"
   
    writelog(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time()))+‘ Start:‘+str_http)
       
    df1=get_58_house(str_http)
    try:
        df1.to_sql(‘house‘, cnx,if_exists=‘append‘)
    except Exception as e:
        ‘‘‘记录异常信息
                    本例使用的是oracle 数据库，默认编码格式为GBK，保存时因为特殊字符，导致保存错误。错误提示如下，需要调整oracle字符集
         oracle 字符集调整为UTF8,
         NLS_LANG: AMERICAN_AMERICA.AL32UTF8
         NLS_CHARACTERSET: UTF8
         NLS_NCHAR_CHARACTERSET: UTF8
                   报错信息为
         UnicodeEncodeError: ‘gbk‘ codec can‘t encode character ‘\\xb2‘ in position 13: illegal multibyte sequence
                     该字符为上标2，平方米          
        ‘‘‘
        writelog(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time()))+‘ Except:‘+str_http)
       
        df1.to_csv(‘record.csv‘,sep=‘,‘, encoding=‘utf-8‘)
        writelog(traceback.format_exc())
       
    writelog(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time()))+‘ End:‘+str_http)
    time.sleep(20)
 
    ##获取其余69页房产数据
    for i in range(2,70+1) :
        try:
            str_http ="http://jn.58.com/ershoufang/pn"+str(i)
            writelog(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time()))+‘ Start:‘+str_http)
           
            df1=get_58_house(str_http)       
            df1.to_sql(‘house‘, cnx,if_exists=‘append‘)
        except Exception as e:
            ##writelog(‘‘.format(‘Save to database Exception‘,":",e)  )
            writelog(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time()))+‘ Except:‘+str_http)
           
            df1.to_csv(‘record.csv‘,sep=‘,‘, encoding=‘utf-8‘)
            writelog(traceback.format_exc())
           
        writelog(time.strftime(‘%Y-%m-%d %H:%M:%S‘,time.localtime(time.time()))+‘ End:‘+str_http)
        time.sleep(20)

    ##关闭数据链接
    cnx.close()

 

跑跑看看是不是程序一切运行正常。