Python怎样写个通用爬虫模板？

其实经常写爬虫的程序员应该都知道，做一个爬虫工作无非就是三个步骤：下载数据、解析数据、保存数据。基本所有爬虫万变不离其宗，都是这样的套路。

下面就是Python写出来一个完整的爬虫需要的几个步骤代码示例，可以供大家参考。

首先是下载数据

def dowmlpad(url, user_agent='wswp', proxy=None, num_retries=2, timeout=5): """ # 支持500错误重试 # 设定用户爬虫ip user_agent # 支持爬虫ip """ print('DownloadURL:',url) #配置用户爬虫ip headers = {'User-agent':user_agent} request = urllib.request.Request(url, headers=headers) #配置 opener = urllib.request.build_opener() #判断是否爬虫ip if proxy: proxy_params = {urllib.parse.urlparse(url).scheme:proxy} opener.add_handler(urllib.request.ProxyHandler(proxy_params)) try: html = opener.open(request, timeout=timeout).read() except urllib.request.URLError as e: print('Download error:',e.reason) html = None if num_retries > 0: if hasattr(e,'code') and 500 <= e.code <600: html = dowmlpad(url, user_agent, num_retries-1) except Exception as e: print('error :',e) html = None return html

其次是解析数据

#编写爬取规则，获得数据def scrape_callback(url,html): csslist = ['span[property = "v:itemreviewed"]', 'span.year', 'strong[property="v:average"]'] try: tree = lxml.html.fromstring(html) row = [tree.cssselect('{0}'.format(field))[0].text for field in csslist] print(url, row) except Exception as e: print("ScrapeCallback error:",e)"""seed_url:种子urllink_regex: 提取链接的正则表达式max_depath：提取链接的深度，默认为2爬虫到达第二场页面后不再提取链接 ，对于种子页面取出的链接页面，就是第二层，scrape_callback：回掉函数"""def link_crawler(seed_url, link_regex, max_depath=2, scrape_callback=None): crawl_queue = [seed_url] #配置爬取队列，其实就是一个存储url的列表 #seen = set(crawl_queue) seens = {seed_url:1} # 循环直到队列为空退出 while crawl_queue: url = crawl_queue.pop() # 移除队列最后一个元素，并返回值 html = dowmlpad(url) # 根据url 下载页面 depth = seens[url] # 获得url深度 print(depth) #获取页面中的链接 for link in get_links(html): if depth != max_depath and re.search(link_regex,link): link = urllib.parse.urljoin(seed_url, link) #组装规范链接 #添加链接到爬取队列中 if link not in seens: seens[link] = depth+1 crawl_queue.append(link) #如果处理回调函数存在，则进行回调处理 if scrape_callback: scrape_callback(url, html)

再次是保存数据

import csvclass ScrapeCallback: def __init__(self): self.writer = csv.writer(open('countries.csv','w')) self.fields = ('name','year','score') self.writer.writerow(self.fields) def __call__(self, url,html): csslist = ['span[property = "v:itemreviewed"]', 'span.year','strong[property="v:average"]'] try: tree = lxml.html.fromstring(html) row = [tree.cssselect('{0}'.format(field))[0].text for field in csslist] self.writer.writerow(row) print(url, row) except Exception as e:\ print("ScrapeCallback error:",e)

最后主函数

if __name__ == '__main__': #测试 send_url = "https://movie.douban.com/" link_regex = '(/subject/[\d]+/)' #获取链接的规则 #使用类的方式来写，下面两个一样结果 link_crawler(send_url,link_regex,max_depath=2, scrape_callback=ScrapeCallback()) #link_crawler(send_url, link_regex, max_depath=2, scrape_callback=scrape_callback)