Python语言使用爬虫IP代码示例

我们在使用爬虫ip对网站进行请求时，经常性会有访问失败的情况，引起这种原因的方式有很多，如果首先排除爬虫IP的问题，那么接下来就要对爬虫程序进行系统的检查，下文就是有关Python语言使用爬虫IP的一些经验可以供大家参考。

Python requests

import requests
targetURL = "http://jshk.com.cn"
proxyAddr = "219.151.125.106:31615"
authKey = "895314XY"
password = "24D6YB309ZCB"
# 账密模式
proxyUrl = "http://%(user)s:%(password)s@%(server)s" % {
"user": authKey,
"password": password,
"server": proxyAddr,
}
proxies = {
"http": proxyUrl,
"https": proxyUrl,
}
resp = requests.get(targetURL, proxies=proxies)
print(resp.text)

Python aiohttp

import aiohttp,asyncio
targetURL = "http://jshk.com.cn"
proxyAddr = "219.151.125.106:31615"
authKey = "895314XY"
password = "24D6YB309ZCB"
# 账密模式
proxyUrl = "http://%(user)s:%(password)s@%(server)s" % {
"user": authKey,
"password": password,
"server": proxyAddr,
}
async def entry():
conn = aiohttp.TCPConnector(ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
async with session.get(targetURL, proxy=proxyUrl) as resp:
body = await resp.read()
print(resp.status)
print(body)
loop = asyncio.get_event_loop()
loop.run_until_complete(entry())
loop.run_forever()

Python urllib2、urllib

import urllib2
targetURL = "http://jshk.com.cn"
proxyAddr = "219.151.125.106:31615"
authKey = "895314XY"
password = "24D6YB309ZCB"
proxyUrl = "http://%(user)s:%(password)s@%(server)s" % {
"user": authKey,
"password": password,
"server": proxyAddr,
}
proxies = urllib2.ProxyHandler({
"http": proxyUrl,
"https": proxyUrl,
})
opener = urllib2.build_opener(proxies)
urllib2.install_opener(opener)
resp = urllib2.urlopen(targetURL).read()
print(resp)