爬取百度贴吧文字内容 方法1: 点击查看代码 **导入urllib库**from urllib import request**导入re正则模块库**import re#指定爬取页数url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90ie=utf-8"try:#指定请求体
方法1:
点击查看代码
**导入urllib库**
from urllib import request
**导入re正则模块库**
import re
#指定爬取页数
url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8"
try:
#指定请求体浏览器UA
headers = {"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"}
req = request.Request(url, headers=headers)
resp = request.urlopen(req)
#把内容进行解码
content = resp.read().decode('utf-8')
print(content)
#定义正则规则进行内容爬取
pattern = re.compile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</a>')
items = re.findall(pattern, content)
#遍历输出内容
for i in items:
print(i[0] + "\t" + i[1])
except request.URLError as e:
if hasattr(e, 'code'):
print(e.code)
if hasattr(e, 'reason'):
print(e.reason)
方法2(封装成方法形式):
点击查看代码
from urllib import request
import re
class TieBa:
# 初始化方法
def __init__(self):
# 链接
self.url = "https://tieba.baidu.com/f?kw=%E6%AE%B5%E5%AD%90&ie=utf-8&pn="
# 请求头
self.headers = {"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"}
# 列表 ,存储解析后的结果
self.stories = []
# 下载页面
def getPage(self, page_number):
try:
# 定义请求
req = request.Request(self.url + str(page_number), headers=self.headers)
# 下载页面
resp = request.urlopen(req)
# 解码
content = resp.read().decode('utf-8')
# 返回结果
return content
except request.URLError as e:
# 打印异常的响应码
if hasattr(e, 'code'):
print(e.code)
if hasattr(e, 'reason'):
print(e.reason)
# 解析页面
# global items
def rexgPage(self, content):
pattern = re.compile(r'<a rel="noopener".*?title=(.*?)\s.*?>(.*?)</ a>')
items = re.findall(pattern, content)
for i in items:
self.stories.append(i[0] + "\t" + i[1])
# 显示解析的内容
def getContent(self):
for i in self.stories:
print(i)
# 调用方法下载然后解析页面 - 显示解析的内容
s1 = TieBa()
print(s1.stories)
s1.rexgPage(s1.getPage(50))
s1.getContent()