scrapy简单分布式爬虫的实现方法

SCHEDULER = "scrapy_redis.scheduler.Scheduler"
 
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
 
REDIS_HOST = '127.0.0.0' #redis的远程ip
REDIS_PORT = 6379 #redis的端口
REDIS_PARAMS = {'password':'密码'} #redis的密码

DOWNLOADER_MIDDLEWARES = {
   'jianshu.middlewares.UserAgentmiddleware': 543,
}

ITEM_PIPELINES = {
    'scrapy_redis.pipelines.RedisPipeline': 300,
    'jianshu.pipelines.MongoPipeline': 800,
}
 
#mongDb
MONGO_URI = '127.0.0.1:27017' #mongodb的ip和端口
MONGO_DATABASE = 'test' #链接的数据库
MONGO_USER = 'admin' #账号
MONGO_PASSWORD = 'admin'#密码

import random
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
 
class UserAgentmiddleware(UserAgentMiddleware):
 
    def process_request(self, request, spider):
        USER_AGENTS =[
        'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+1.1.4322;+TencentTraveler)',
        'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1;+.NET+CLR+2.0.50727;+.NET+CLR+3.0.4506.2152;+.NET+CLR+3.5.30729)',
        'Mozilla/5.0+(Windows+NT+5.1)+AppleWebKit/537.1+(KHTML,+like+Gecko)+Chrome/21.0.1180.89+Safari/537.1',
        'Mozilla/4.0+(compatible;+MSIE+6.0;+Windows+NT+5.1;+SV1)',
        'Mozilla/5.0+(Windows+NT+6.1;+rv:11.0)+Gecko/20100101+Firefox/11.0',
        'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+SV1)',
        'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+GTB7.1;+.NET+CLR+2.0.50727)',
        'Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0;+KB974489)',
        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
        'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
        "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E)",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
        "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"
        ]
        agent = random.choice(USER_AGENTS)
        request.headers["User-Agent"] = agent

import scrapy
 
 
class JianshuItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    html = scrapy.Field()
    url  = scrapy.Field()

# -*- coding: utf-8 -*-
 
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
 
import pymongo
 
class MongoPipeline(object):
 
    collection_name = 'scrapy_items'
 
    def __init__(self, mongo_uri, mongo_db,mongo_user,mongo_password):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
        self.mongo_user = mongo_user
        self.mongo_password = mongo_password
 
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items'),
            mongo_user = crawler.settings.get('MONGO_USER'),
            mongo_password = crawler.settings.get('MONGO_PASSWORD'),
        )
 
    def open_spider(self, spider):
        self.client = pymongo.MongoClient('mongodb://%s:%s@%s/admin' % (self.mongo_user,self.mongo_password,self.mongo_uri))
        self.db = self.client[self.mongo_db]
 
    def close_spider(self, spider):
        self.client.close()
 
    def process_item(self, item, spider):
        self.db[self.collection_name].insert(dict(item))
        return item

# -*- coding: utf-8 -*-
import scrapy
from jianshu.items import JianshuItem
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
 
 
class Spider1Spider(CrawlSpider):
    name = "spider1"
    allowed_domains = ["jianshu.com"]
    start_urls = (
        'http://www.jianshu.com/',
    )
 
    rules = (
        # 提取匹配 'category.php' (但不匹配 'subsection.php') 的链接并跟进链接(没有callback意味着follow默认为True)
        Rule(LinkExtractor(allow=('/c/', '/u/',) )),
 
        # 提取匹配 'item.php' 的链接并使用spider的parse_item方法进行分析
        Rule(LinkExtractor(allow=('/p/', )), callback='parse_item',follow=True),
    )
 
 
    def parse_item(self,response):
        html = response.body
        url = response.url
 
        item = JianshuItem()
        item['url'] = url
        item['html'] = html
        return item