NIPS 全文下载

有些收集癖，所以这次打算把NIPS上历年的论文都下载下来。尝试通过python直接下载，不过发现很慢，所以想到，那就直接先爬取所有的下载链接，分好文件夹，然后在手动对应不同的年份将其复制到迅雷中下载（目前一共开了29期，所以手动工作还好）

代码及解释如下：

# -*- coding: utf-8 -*-
""" Created on Sat Sep 9 19:10:39 2017 @author: shouhuxianjian """
'''导入包'''
import os
import re
#import wget
import os.path as osp
import requests
from bs4 import BeautifulSoup as bs
url0 = 'http://papers.nips.cc/'
'''下载首页'''
html0 = requests.get(url = url0)
html0 = bs(html0.text,'html5lib')

gResDir = r'e:\NIPS'#存储的位置
NIPSTimes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
             11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
             21, 22, 23, 24, 25, 26, 27, 28, 29]#这条没用到
'''获取所有会议次数 '''
books_hrefs = [(li.text,li.find('a')['href']) for li in \
                 html0.find_all('li',text=re.compile('Advances'))]
books_hrefs = [(book.replace(' ','_'), url0.rstrip('/')+href ) for book,href in \
                books_hrefs]
'''创建文件夹函数 '''
def makedirs(indir):
    try:
        os.makedirs(indir)
    except Exception as e:
        pass
'''为每次会议单独创建一个文件夹'''
[makedirs(osp.join(gResDir,indir)) for indir,_ in books_hrefs]

invalidChars = re.compile('[;?\*|"<>:/]')#windows下文件夹命名的非法字符

cur = 14
'''读取每次会议，并读取该次会议下面的paper链接 '''
for indb,(book,href) in enumerate(books_hrefs):

    if indb < cur: continue
    html1 = requests.get(url = href)
    html1 = bs(html1.text,'html5lib')
    papers_hrefs = [(li.text,li.find('a')['href']) for li in html1.find_all('li') if\
                     '/paper/' in li.find('a')['href'] ]
    papers_hrefs = [(invalidChars.sub(' ',paper),url0.rstrip('/')+hrefPaper ) for \
                     paper,hrefPaper in papers_hrefs]
    resPath = osp.join(gResDir,book)

    '''读取该次会议已经下载过paper的链接，防止多次下载，即跳过之前下载的部分，从而有断点续下载功能 '''
    papersDi = {line.strip():1 for line in open(osp.join(resPath,'book.txt'),encoding = 'utf-8')} \
               if osp.exists( osp.join(resPath,'book.txt')) else {}

    ''' 读取该次会议下paper的链接，并将其中的pdf和supplemental链接保存到文件中'''      
    for ind,(paper,hrefPapers) in enumerate(papers_hrefs):

        if paper in papersDi: continue
        print('book:[{}/{}] [{}]'.format(indb+1,len(books_hrefs),book))
        print('paper:[{}/{}] [{}]'.format(ind+1,len(papers_hrefs),paper))
        print('='*50)

        '''下面设置超时30秒，如果爆出超时异常，中断当前程序然后重开就行'''
        html2 = requests.get(url = hrefPapers,timeout = 30)
        html2 = bs(html2.text,'html5lib')

        '''将未下载pdf的paper名字保存到文件cannotDownload.txt中 '''
        try:
            hrefPDF = html2.find('a',text = '[PDF]')['href']
        except Exception as e:
            with open(osp.join(resPath,'cannotDownload.txt'),'a',encoding = 'utf-8') as fa:
                fa.write(paper+'\n')
            continue

        '''有些paper并没有supplemental '''
        supplemental = html2.find('a',text = '[Supplemental]')
        hrefSupplemental = supplemental['href'] if supplemental else ''

        '''如下代码运行时下载速度慢，所以才将下载链接收集起来，交给迅雷等下载 '''
# pdf = wget.download(url = url0.rstrip('/')+hrefPDF,
# out = osp.join(resPath,paper))
# pdfContent = requests.get(url = url0.rstrip('/')+hrefPDF )

# with open(osp.join(resPath,paper.split(',')[0])+'.pdf','wb') as fw:
# fw.write(pdfContent.content)

        '''将pdf下载链接写入文件urls.txt '''
        with open(osp.join(resPath,'urls.txt'),'a',encoding = 'utf-8') as fw:
            fw.write(url0.rstrip('/')+hrefPDF+'\n')
        '''处理对应supplemental ''' 
        if hrefSupplemental:
            suppFormat = hrefSupplemental.split('.')[-1]
# suppContent = requests.get(url = url0.rstrip('/')+hrefSupplemental )
# with open(osp.join(resPath,paper.split(',')[0])+'.'+suppFormat,'wb') as fw:
# fw.write(suppContent.content)
            with open(osp.join(resPath,'urls.txt'),'a',encoding = 'utf-8') as fw:
                fw.write(url0.rstrip('/')+hrefSupplemental+'\n')
        '''最后将当前paper名字写入文件book.txt用于确保上述行为运行结束，且保证断点续下载功能 '''
        with open(osp.join(resPath,'book.txt'),'a',encoding = 'utf-8') as fa:
            fa.write(paper+'\n')

结果如下图所示：
1 - 按照会议次数创建文件夹

2 - 每次会议文件夹下生成文件

3 - urls.txt中下载链接

已经将1988-2016 共29次会议的下载链接放在这里了