以爬取斗鱼直播上的信息为例：

URL地址：http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset=0

爬取字段：房间ID、房间名、图片链接、存储在本地的图片路径、昵称、在线人数、城市

1.items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class DouyuspiderItem(scrapy.Item):
    # define the fields for your item here like:
    # 房间ID
    room_id = scrapy.Field()
    # 房间名
    room_name = scrapy.Field()
    # 图片链接
    vertical_src = scrapy.Field()
    # 存储图片的本地地址
    image_path = scrapy.Field()
    # 昵称
    nickname = scrapy.Field()
    # 在线人数
    online = scrapy.Field()
    # 城市
    anchor_city = scrapy.Field()

 2.spiders/douyu.py

# -*- coding: utf-8 -*-
import scrapy
from douyuSpider.items import DouyuspiderItem
import json

class DouyuSpider(scrapy.Spider):
    name = 'douyu'
    allowed_domains = ['capi.douyucdn.cn']
    url = 'http://capi.douyucdn.cn/api/v1/getVerticalRoom?limit=20&offset='
    offset = 0
    start_urls = [url + str(offset)]

    def parse(self, response):
        # 是否爬取下一页的标记
        next_flag = False
        data = json.loads(response.text)["data"]
        for each in data:
            item = DouyuspiderItem()
            # 房间ID
            item['room_id'] = each["room_id"]
            # 房间名
            item['room_name'] = each["room_name"]
            # 图片链接
            item['vertical_src'] = each["vertical_src"]
            # 昵称
            item['nickname'] = each["nickname"]
            # 在线人数
            item['online'] = each["online"]
            # 城市
            item['anchor_city'] = each["anchor_city"]

            next_flag = True

            yield item

        # 判断是否继续爬取下一页
        if next_flag:
            self.offset += 20
            yield scrapy.Request(self.url + str(self.offset), callback = self.parse)

 3.pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import json
import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.utils.project import get_project_settings
import os

# 存储信息的json文件中间件
class DouyuspiderPipeline(object):
    def __init__(self):
        self.file = open("斗鱼.json", "w", encoding = "utf-8")
        self.first_flag = True

    def process_item(self, item, spider):
        if self.first_flag:
            self.first_flag = False
            content = "[\n" + json.dumps(dict(item), ensure_ascii = False)
        else:
            content = ",\n" + json.dumps(dict(item), ensure_ascii = False)
        self.file.write(content)

        return item

    def close_spider(self, spider):
        self.file.write("\n]")
        self.file.close()

# 下载图片的中间件
class ImagesPipeline(ImagesPipeline):
    IMAGES_STORE = get_project_settings().get("IMAGES_STORE")

    def get_media_requests(self, item, info):
        image_url = item["vertical_src"]

        yield scrapy.Request(image_url)

    def item_completed(self, results, item, info):
        # 固定写法，获取图片路径，同时判断这个路径是否正确，如果正确，就放到 image_path里，ImagesPipeline源码剖析可见
        image_path = [x["path"] for ok, x in results if ok]

        os.rename(self.IMAGES_STORE + "/" + image_path[0], self.IMAGES_STORE + "/" + item["nickname"] + ".jpg")
        item["image_path"] = self.IMAGES_STORE + item["nickname"] + ".jpg"

        return item

 4.settings.py

# -*- coding: utf-8 -*-

# Scrapy settings for dou