вот кусок моего когда:
def parse(self, response, **kwargs):
print("procesing:"+response.url)
def __init__(self, *args, **kwargs):
super(CatalogSpider, self).__init__(*args, **kwargs)
self.proxy_pool = ["51.68.141.31:80","193.149.225.220:80","104.248.90.212:80","35.198.167.227:80","85.214.65.246:80","91.185.222.11:80","95.138.193.63:80","191.101.39.225:80","75.119.159.78:80","64.227.122.38:8080","102.129.255.223:80",'34.91.22.51:3128','3.125.229.175:80','52.47.137.181:80','65.21.3.120:80','13.36.167.122:8080','46.101.254.73:80','23.251.138.105:8080','178.254.42.75:80','5.135.160.119:8080','15.237.64.173:80','54.220.128.74:80','51.77.123.247:80','51.75.206.209:80','34.249.135.31:3128','85.255.1.116:80','135.181.198.130:80','51.158.112.32:3128','212.85.66.140:8193','207.180.239.232:80','13.36.102.214:80','167.99.255.129:80','217.182.169.170:80','51.91.157.66:80','193.149.225.188:80','195.235.90.29:80','144.91.97.235:80','91.232.196.77:80','137.74.112.21:80','163.172.98.25:80','167.172.180.46:39898','62.83.221.231:80','5.28.128.60:80','188.130.255.17:80','149.54.11.66:80','116.202.101.41:80','80.39.228.110:80','188.130.255.12:80','188.130.255.9:80','84.54.82.234:3128']
for links in response.css('div.col-three'):
for i in links.css('a::attr(href)').extract():
yield scrapy.Request(self.etalon+i, callback=self.pars2)
def pars2(self, response, **kwargs):
x = response.css('#content ul li a::attr(href)').extract()
for i in x:
yield scrapy.Request(self.etalon+i, callback=self.pars3)
def pars3(self, response, **kwargs):
x = response.css('#content td a::attr(href)').extract()
for i in x:
yield scrapy.Request(self.etalon+i, callback=self.pars4)
