Python爬虫之利用BeautifulSoup爬取豆瓣小说（三）——将小说信息写入文件

1 #-*-coding:utf-8-*-
2 import urllib2
3 from bs4 import BeautifulSoup
4
5 class dbxs:
6
7 def __init__(self):
8 self.pageIndex = 0
9 self.enable = True
10 self.file = None
11 self.content = []
12
13
14 #获取html页面的内容
15 def getPage(self, pageIndex):
16 try:
17 #设置代理ip
18 enable_proxy = True
19 proxy_handler = urllib2.ProxyHandler({'Http': '113.118.170.230:808'})
20 null_proxy_handler = urllib2.ProxyHandler({})
21 if enable_proxy:
22 opener = urllib2.build_opener(proxy_handler)
23 else:
24 opener = urllib2.build_opener(null_proxy_handler)
25 urllib2.install_opener(opener)
26 #获得页面响应的内容
27 url = 'https://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book' + "?start=" + str(pageIndex)
28 #设置请求头部信息，模拟浏览器的行为
29 my_headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0)'}
30 request = urllib2.Request(url, headers = my_headers)
31 response = urllib2.urlopen(request)
32 return response.read()
33 except urllib2.URLError, e:
34 if hasattr(e, "code"):
35 print e.code
36 if hasattr(e, "reason"):
37 print e.reason
38 return None
39
40 #过滤查找这一页的小说名字，信息和评分
41 def getContent(self, pageIndex, content):
42 pageCode = self.getPage(pageIndex)
43 soup = BeautifulSoup(pageCode, 'html.parser')
44 #在获得相应的内容中找出所有标签为<dd>的内容（里面包含了我们需要的小说信息）
45 contents = soup.find_all('dd')
46
47 if contents:
48 for item in contents:
49 title = item.find(class_ = 'title').string.encode('utf-8')
50 info = item.find(class_ = 'desc').string.strip().encode('utf-8')
51 rate = item.find(class_ = 'rating_nums')
52 #通过试验，我们发现某一页可能存在小说没有评分，如果我们不判断rate，那么可能就出现报错
53 if rate:
54 rates = rate.string.encode('utf-8')
55 content.append([title, info, rates])
56
57 else:
58 content.append([title, info])
59 #如果页面不包含<dd>标签，我们应该停止
60 else:
61 print u"所有页面已加载完"
62 self.enable = False
63
64 return content
65
66
67
68 #写入文件
69 def writeData(self, content):
70 self.file = open("bdxs.txt", "w+") #必须在for循环外面，不然每一次写入都会覆盖之前的数据
71 for item in content:
72 if len(item) == 3:
73 self.file.write(item[0] + "\n")
74 self.file.write(item[1] + "\n")
75 self.file.write(u"评分:" + item[2] + "\n\n")
76 else:
77 self.file.write(item[0] + "\n")
78 self.file.write(item[1] + "\n")
79 self.file.write("========================================\n\n")
80
81
82 #创建一个开始方法
83 def start(self):
84 x = 1
85 while self.enable == True:
86 content = self.getContent(self.pageIndex, self.content)
87 if self.enable == True:
88 print "正在写入第%s页..." %x
89 self.writeData(content)
90 self.pageIndex += 15
91 x += 1
92
93
94 DBXS = dbxs()
95 DBXS.start()

这段代码我还没理解透彻，比如每一页的小说信息写入完成后，怎么在后面加上第几页，后期我将继续完善它。