怎么写爬虫，怎么找网站练手抓取链家中原安居客我爱我家，今年5月份开始写论文啦！！！

Posted 花神华安

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了怎么写爬虫，怎么找网站练手抓取链家中原安居客我爱我家，今年5月份开始写论文啦！！！相关的知识，希望对你有一定的参考价值。

 

 

//设置请求时间

string html = string.Empty;
try
{
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;//模拟请求
request.Timeout = 30 * 1000;//设置30s的超时
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36";
request.ContentType = "text/html; charset=utf-8";
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse) //发起请求
{
if (response.StatusCode != HttpStatusCode.OK)
{
log.Error("抓取{0}地址返回失败，响应状态为{1}", url, response.StatusCode);
}
else
{
try
{
StreamReader sr = new StreamReader(response.GetResponseStream(), encode);
html = sr.ReadToEnd();//读取数据
sr.Close();
}
catch (Exception ex)
{
log.Error("抓取{0}失败", url, ex);
html = null;
}
}
}
}
catch (Exception ex)
{
log.Error("抓取{0}出现异常", url, ex);
html = null;
}
return html;

 

//抓取链家、中原、安居客、我爱我家

string html = HttpHelper.DownloadUrl(pageurl);
if (html == null)
{
return houseList;
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
//链家
string psht = @"//*[@class=‘content‘]/div[@class=‘leftContent‘]/ul[@class=‘listContent‘]/li[@class=‘clear xiaoquListItem‘]";
HtmlNodeCollection noneNodeList = doc.DocumentNode.SelectNodes(psht);
if (noneNodeList == null)
{
log.ErrorAsync("数据为空！");
return houseList;
}
foreach (var item in noneNodeList)
{
TrojanHorse house = new TrojanHorse();
HtmlDocument docChild = new HtmlDocument();
docChild.LoadHtml(item.OuterHtml);
//链家
string urlPath = @"//*[@class=‘info‘]/div[@class=‘title‘]/a";
HtmlNode urlNode = docChild.DocumentNode.SelectSingleNode(urlPath);
string tsct = urlNode.InnerText;//小区名称
//链家 [@class=‘xiaoquListItemRight‘]/div
string strs = @"//*[@class=‘xiaoquListItemPrice‘]/div[@class=‘totalPrice‘]/span";
HtmlNode urlNodes = docChild.DocumentNode.SelectSingleNode(strs);
string s = "";
if (urlNodes == null)
{
s = "null";
}
else
{
s = urlNodes.InnerText;
}
//链家
string strst = @"//*[@class=‘xiaoquListItemRight‘]/div[@class=‘xiaoquListItemSellCount‘]/a[@class=‘totalSellCount‘]/span";
HtmlNode urlNodest = docChild.DocumentNode.SelectSingleNode(strst);
string st = "";
if (urlNodest == null)
{
st = "null";
}
else
{
st = urlNodest.InnerText;
}
#region
//string tscts = s.Replace(" ", "");
//string tsctst = tscts.Substring(0, 8);
//string tsctsb = tscts.Substring(tscts.Length - 7, 7);
// string rsf = s;
//string zf = tsctsb.Substring(0, 5);//租房
#endregion
house.title = tsct;
house.price = s;
house.remark = st;
houseList.Add(house);
#region
//house.StaffName = urlNode.Attributes["title"].Value; //经纪人姓名
//string companyPath = "//*[@class=‘jjr-info‘]/p[@class=‘jjr-desc mg-top‘]/a[position()<2]";
//HtmlNode companyNode = docChild.DocumentNode.SelectSingleNode(companyPath);
//if (companyNode == null)
//{
// continue;
//}
//house.Company = companyNode.InnerText; //中介公司
//string telPath = "//*[@class=‘jjr-side‘]";
//HtmlNode telNode = docChild.DocumentNode.SelectSingleNode(telPath);
//if (telNode == null)
//{
// continue;
//}
//string telstr = telNode.InnerText.Trim();
//house.Mobile = telstr; //经纪人电话
//house.CityCode = citycode; //城市代号
//house.CreateTime = DateTime.Now;
//var flag = houseList.Where(x => x.Mobile == telstr).FirstOrDefault(); //有重复的手机号不添加
//if (flag == null)
//{
// houseList.Add(house);
//}
#endregion
}
}

/// <summary>
/// 抓取每一页的数据
/// </summary>
/// <param name="pageurl"></param>
/// <returns></returns>
private static List<TrojanHorse> GetTrojanHorseList(string pageurl)
{
List<TrojanHorse> houseList = new List<TrojanHorse>();
try
{
string html = HttpHelper.DownloadUrl(pageurl);
if (html == null)
{
return houseList;
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(html);
//链家
string psht = @"//*[@class=‘content‘]/div[@class=‘leftContent‘]/ul[@class=‘listContent‘]/li[@class=‘clear xiaoquListItem‘]";
HtmlNodeCollection noneNodeList = doc.DocumentNode.SelectNodes(psht);
if (noneNodeList == null)
{
log.ErrorAsync("数据为空！");
return houseList;
}
foreach (var item in noneNodeList)
{
TrojanHorse house = new TrojanHorse();
HtmlDocument docChild = new HtmlDocument();
docChild.LoadHtml(item.OuterHtml);
//链家
string urlPath = @"//*[@class=‘info‘]/div[@class=‘title‘]/a";
HtmlNode urlNode = docChild.DocumentNode.SelectSingleNode(urlPath);
string tsct = urlNode.InnerText;//小区名称
//链家 [@class=‘xiaoquListItemRight‘]/div
string strs = @"//*[@class=‘xiaoquListItemPrice‘]/div[@class=‘totalPrice‘]/span";
HtmlNode urlNodes = docChild.DocumentNode.SelectSingleNode(strs);
string s = "";
if (urlNodes == null)
{
s = "null";
}
else
{
s = urlNodes.InnerText;
}
//链家
string strst = @"//*[@class=‘xiaoquListItemRight‘]/div[@class=‘xiaoquListItemSellCount‘]/a[@class=‘totalSellCount‘]/span";
HtmlNode urlNodest = docChild.DocumentNode.SelectSingleNode(strst);
string st = "";
if (urlNodest == null)
{
st = "null";
}
else
{
st = urlNodest.InnerText;
}
#region
//string tscts = s.Replace(" ", "");
//string tsctst = tscts.Substring(0, 8);
//string tsctsb = tscts.Substring(tscts.Length - 7, 7);
// string rsf = s;
//string zf = tsctsb.Substring(0, 5);//租房
#endregion
house.title = tsct;
house.price = s;
house.remark = st;
houseList.Add(house);
#region
//house.StaffName = urlNode.Attributes["title"].Value; //经纪人姓名
//string companyPath = "//*[@class=‘jjr-info‘]/p[@class=‘jjr-desc mg-top‘]/a[position()<2]";
//HtmlNode companyNode = docChild.DocumentNode.SelectSingleNode(companyPath);
//if (companyNode == null)
//{
// continue;
//}
//house.Company = companyNode.InnerText; //中介公司
//string telPath = "//*[@class=‘jjr-side‘]";
//HtmlNode telNode = docChild.DocumentNode.SelectSingleNode(telPath);
//if (telNode == null)
//{
// continue;
//}
//string telstr = telNode.InnerText.Trim();
//house.Mobile = telstr; //经纪人电话
//house.CityCode = citycode; //城市代号
//house.CreateTime = DateTime.Now;
//var flag = houseList.Where(x => x.Mobile == telstr).FirstOrDefault(); //有重复的手机号不添加
//if (flag == null)
//{
// houseList.Add(house);
//}
#endregion
}
}


#region
// {
// string html = HttpHelper.DownloadUrl(pageurl);
// if (html == null)
// {
// return houseList;
// }
// HtmlDocument doc = new HtmlDocument();
// doc.LoadHtml(html);
// //*[@id="pebpwbwege"]
// //链家
// //string psht = @"//*[@class=‘leftContent‘]/ul[@class=‘listContent‘]/li[@class=‘clear xiaoquListItem‘]";

// //中原
// string psht = @"//*[@class=‘section-wrap section-houselists‘]/div[@class=‘section‘]/div[@class=‘house-item clearfix‘]";
// HtmlNodeCollection noneNodeList = doc.DocumentNode.SelectNodes(psht);
// if (noneNodeList == null)
// {
// log.ErrorAsync("数据为空！");
// return houseList;
// }
// foreach (var item in noneNodeList)
// {
// TrojanHorse house = new TrojanHorse();
// HtmlDocument docChild = new HtmlDocument();
// docChild.LoadHtml(item.OuterHtml);
// //docChild.LoadHtml();
// //链家
// //string urlPath = @"//*[@class=‘info‘]/div[@class=‘title‘]/a";
// //中原
// string urlPath = @"//*[@class=‘item-info fl‘]/h4/a";
// HtmlNode urlNode = docChild.DocumentNode.SelectSingleNode(urlPath);
// //docChild.DocumentNode.SelectSingleNode(str);
// string tsct = urlNode.InnerText;//小区名称
// //链家
// //string strs = @"//*[@class=‘xiaoquListItemRight‘]/div[@class=‘xiaoquListItemPrice‘]/div[@class=‘totalPrice‘]/span";

// //中原
// string strs = @"//*[@class=‘item-pricearea fr‘]/p[@class=‘tc f666 f12 mt_10‘]/a";
// HtmlNode urlNodes = docChild.DocumentNode.SelectSingleNode(strs);
// string s = "";
// if (urlNodes == null)
// {
// s = "null";
// }
// else
// {
// s = urlNodes.InnerText;

// }
// //链家
// //string strst = @"//*[@class=‘xiaoquListItemRight‘]/div[@class=‘xiaoquListItemSellCount‘]/a[@class=‘totalSellCount‘]/span";
// //中原
// string strst = @"//*[@class=‘item-pricearea fr‘]/p[@class=‘price-nub cRed tc‘]/span";
// HtmlNode urlNodest = docChild.DocumentNode.SelectSingleNode(strst);
// string st = "";
// if (urlNodest == null)
// {
// st = "null";
// }
// else
// {
// st = urlNodest.InnerText;

// }

// //string tscts = s.Replace(" ", "");
// //string tsctst = tscts.Substring(0, 8);
// //string tsctsb = tscts.Substring(tscts.Length - 7, 7);
// // string rsf = s;
// //string zf = tsctsb.Substring(0, 5);//租房
// house.title = tsct;
// house.price = s;
// house.remark = st;
// houseList.Add(house);

 

// //house.StaffName = urlNode.Attributes["title"].Value; //经纪人姓名
// //string companyPath = "//*[@class=‘jjr-info‘]/p[@class=‘jjr-desc mg-top‘]/a[position()<2]";
// //HtmlNode companyNode = docChild.DocumentNode.SelectSingleNode(companyPath);
// //if (companyNode == null)
// //{
// // continue;
// //}
// //house.Company = companyNode.InnerText; //中介公司
// //string telPath = "//*[@class=‘jjr-side‘]";
// //HtmlNode telNode = docChild.DocumentNode.SelectSingleNode(telPath);
// //if (telNode == null)
// //{
// // continue;
// //}
// //string telstr = telNode.InnerText.Trim();
// //house.Mobile = telstr; //经纪人电话
// //house.CityCode = citycode; //城市代号
// //house.CreateTime = DateTime.Now;
// //var flag = houseList.Where(x => x.Mobile == telstr).FirstOrDefault(); //有重复的手机号不添加
// //if (flag == null)
// //{
// // houseList.Add(house);
// //}
// }
//}
#endregion
catch (Exception ex)
{
log.ErrorAsync("服务器异常,异常信息：" + ex.Message);
}

以上是关于怎么写爬虫，怎么找网站练手抓取链家中原安居客我爱我家，今年5月份开始写论文啦！！！的主要内容，如果未能解决你的问题，请参考以下文章

Python爬虫可以爬取啥

上海二手房8月排名：链家悟空找房中原太平洋我爱我家易居房天下iwjw房多多房好多q房网

python爬取安居客二手房网站数据（转）

上海租房攻略

Python开发爬虫之BeautifulSoup解析网页篇：爬取安居客网站上北京二手房数据

安居客二手房爬虫-微信提醒合适房源！