python爬虫单线程与多线程区别

之前有人请我帮忙写一个有关招聘的爬虫，一开始先是单线程，结果显而易见非常慢，后来改了多线程，速度杠杠的。

1、单线程

import urllibimport urllib.requestimport requestsimport xlwtimport reimport string def set_style(name,height,bold=False): style = xlwt.XFStyle() # 初始化样式 font = xlwt.Font() # 为样式创建字体 http://font.name = name # 'Times New Roman' font.bold = bold font.color_index = 4 font.height = height style.font = font return style index=0r=0wdk=xlwt.Workbook() sheet1=wdk.add_sheet('sheet1',cell_overwrite_ok=True) #excel单元格名字row=['招聘职位','月薪','学历要求','工作经验','应聘要求','工作单位']for i in row: sheet1.write(r,index,i) index+=1r+=1import datetime starttime = datetime.datetime.now() for pn in range(0,200,20): url = 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98%E5%B7%A5%E7%A8%8B%E5%B8%88&city_sug=%E9%95%BF%E6%B2%99&&detailmode=close&rn=20&pn='+str(pn)#打开网页 wp = urllib.request.urlopen(url) content = wp.read() hrefPatten=r"(href='.{0,200}数据挖掘工程师')" href=hrefPatten.encode("utf-8") hrefC = re.findall(href, content, re.S) #返回所有匹配正则表达式的值于列表中 hre=['']*len(hrefC) for i in range (len(hrefC)): hre[i]=urllib.parse.unquote(str(hrefC[i].decode('utf-8')).replace("href='","http://zhaopin.baidu.com")[:-1])#将href转化为网址 for i in range (len(hre)): a=requests.get(hre[i]) content1=a.content.decode('utf-8') pattern=('<span class="title line-clamp1">(.*?)</span>.*?class="salary">(.*?)</span>.*?class="xueli"></span>(.*?)</li>.*?class="minge"></span>(.*?)</li>.*?class="duty duty-box">(.*?)</p>.*?class="line-clamp2">(.*?)</p>') items=re.findall(pattern, content1, re.S)#抓取招聘信息 for item in items: flag=0 for _item in item: if _item=="": flag=1 if flag==0: for k in range(0,6): sheet1.write(r,k,item[k]) r+=1 wdk.save('/home/y.xls')endtime = datetime.datetime.now() print((endtime - starttime).seconds)

用时80S完成。

2、多线程

import urllibimport urllib.requestimport requestsimport xlwtimport reimport stringimport threadingfrom time import ctime,sleep def set_style(name,height,bold=False): style = xlwt.XFStyle() # 初始化样式 font = xlwt.Font() # 为样式创建字体 http://font.name = name # 'Times New Roman' font.bold = bold font.color_index = 4 font.height = height style.font = font return style import datetimestarttime = datetime.datetime.now() index=0global rr=0wdk=xlwt.Workbook() sheet1=wdk.add_sheet('sheet1',cell_overwrite_ok=True) #excel单元格名字row=['招聘职位','月薪','学历要求','工作经验','应聘要求','工作单位']for i in row: sheet1.write(r,index,i) index+=1r+=1inte=[]def f(pn): url = 'http://zhaopin.baidu.com/quanzhi?tid=4139&ie=utf8&oe=utf8&query=%E6%95%B0%E6%8D%AE%E6%8C%96%E6%8E%98%E5%B7%A5%E7%A8%8B%E5%B8%88&city_sug=%E9%95%BF%E6%B2%99&&detailmode=close&rn=20&pn='+str(pn)#打开网页 wp = urllib.request.urlopen(url) content = wp.read() hrefPatten=r"(href='.{0,200}数据挖掘工程师')" href=hrefPatten.encode("utf-8") hrefC = re.findall(href, content, re.S) #返回所有匹配正则表达式的值于列表中 hre=['']*len(hrefC) for i in range (len(hrefC)): hre[i]=urllib.parse.unquote(str(hrefC[i].decode('utf-8')).replace("href='","http://zhaopin.baidu.com")[:-1])#将href转化为网址 for i in range (len(hre)): a=requests.get(hre[i]) content1=a.content.decode('utf-8') pattern=('<span class="title line-clamp1">(.*?)</span>.*?class="salary">(.*?)</span>.*?class="xueli"></span>(.*?)</li>.*?class="minge"></span>(.*?)</li>.*?class="duty duty-box">(.*?)</p>.*?class="line-clamp2">(.*?)</p>') items=re.findall(pattern, content1, re.S)#抓取招聘信息 for item in items: flag=0 for _item in item: if _item=="": flag=1 if flag==0: inte.append(item) endtime = datetime.datetime.now() print((endtime - starttime).seconds) threads = []for pn in range(0,200,20): threads.append(threading.Thread(target=f,args=(pn,))) sleep(0.1) import datetime starttime = datetime.datetime.now() if __name__ == '__main__': for t in threads: t.setDaemon(True) t.start() sleep(15) ind=1 for item in inte: for k in range(0,6): sheet1.write(ind,k,item[k]) ind+=1 wdk.save('/home/t.xls')

用时只要15S。