#!/usr/bin/python
# -*- coding: utf-8 -*-
from scrapy.crawler import Crawler
from scrapy.utils.project import get_project_settings
from twisted.internet import reactor
# Импортирем пауков
from spiders.newenglandfilm import NewenglandFilm
from spiders.mandy import Mandy
from spiders.productionhub import ProductionHub
from spiders.craiglist import Craiglist
from spiders.my_settings import options
# Передаем настройки
settings = get_project_settings()
settings.overrides.update(options)
# Запускаем четыре паука по очереди
crawler = Crawler(settings)
crawler.configure()
crawler.crawl(NewenglandFilm())
crawler.start()
crawler = Crawler(settings)
crawler.configure()
crawler.crawl(Mandy())
crawler.start()
crawler = Crawler(settings)
crawler.configure()
crawler.crawl(ProductionHub())
crawler.start()
crawler = Crawler(settings)
crawler.configure()
crawler.crawl(Craiglist())
crawler.start()
# Запускаем реактор
reactor.run()for num in range(0, 4):
while b[num] <= e[num]:
# yield
b[num] += 1
if b[num] <= e[num]:
b[num + 1] = beggining[num + 1]