python scrapy 下载网站数据

主要源码如下：

import scrapy
from pc.items import FileItem
import json
import math
import datetime

class xxSpider(scrapy.Spider):
name = 'xx'
allowed_domains = ['xx.com']
#offset = 1

#max_page=10 #抓取的最大页数
baseURL = 'https://www.xxx.com/js/piaofu.html'
start_urls = [baseURL]

def parse(self, response):
url_list = response.xpath("//div[@class='list-pngjs']/dl/dd/a/@href").extract()
for url in url_list:
href = 'https://www.xx.com' + url
print("href" * 30)
print(href)
print("href" * 30)
if type(href) == str:
yield scrapy.Request(
href,
callback=self.parse_detail
)

if len(response.xpath("//div[@class='dede_pages']/ul/li[@class='thisclass']/following-sibling::li")):
url = response.xpath("//div[@class='dede_pages']/ul/li[@class='thisclass']/following-sibling::li/a/@href").extract()[0]
url = 'https://www.xx.com' + url
print("0" * 30)
print(url)
print("0" * 30)
yield scrapy.Request(url, callback=self.parse)


def parse_detail(self,response):
pic_id = response.xpath("//div[@id='l']/div[@class='content-a']/div[@class='xiazai']/a[@class='bt-blue js-download']/@data-fileid").extract()[0]
url='https://www.xxx.com/js/d'+pic_id+'.zip'
# fileUrl为相对路径时，可用response.urljoin(url)进行拼接
item = FileItem(file_urls=[url])
yield item # 注意：此处为yield，不是return

下载源码