phantomjs 设置代理

Posted 2020-09-05 niansi

phantomjs 可通过以下设置代理ip

#coding=utf-8
import os
import re
import time
import requests
from scrapy.selector import htmlXPathSelector
from scrapy.http import HtmlResponse
from selenium import webdriver
from selenium.webdriver.common.proxy import ProxyType

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

import warnings
warnings.filterwarnings("ignore")

if __name__ == \'__main__\':
PATH_PHANTOMJS=r\'D:\\phantomjs\\bin\\phantomjs.exe\'
browser=webdriver.PhantomJS(PATH_PHANTOMJS)
#真实ip
browser.get(\'http://icanhazip.com/\') #获取ip网址
response = HtmlResponse(url=\'\',body=str(browser.page_source))
hxs = HtmlXPathSelector(response)
print \'your ip is:\',\'\'.join(hxs.select(\'//text()\').extract()).strip()
#代理ip
proxy=webdriver.Proxy()
proxy.proxy_type=ProxyType.MANUAL
proxy.http_proxy=\'220.248.229.45:3128\'
#将代理设置添加到webdriver.DesiredCapabilities.PHANTOMJS中
proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
browser.get(\'http://icanhazip.com/\') #获取ip网址
response = HtmlResponse(url=\'\',body=str(browser.page_source))
hxs = HtmlXPathSelector(response)
print \'your proxy ip is:\',\'\'.join(hxs.select(\'//text()\').extract()).strip()

经测试，实际有效，截图如下：

后期可以考虑将此方法运用至下载中，减少爬虫被封的几率。