Зависает скрипт. Всегда на одной и той же точке (11% / 91/817 запросов).
import time
import aiohttp
import asyncio
from collections import Counter
from fake_useragent import UserAgent
from tqdm import tqdm
async def answer(link: str, num_of_requests: int):
RATE_LIMIT = 6
# TIME_TO_SLEEP = 0
proxies = [('http://109.677.467.275:3000', aiohttp.BasicAuth('оiVszD', '8gsdfASJzTy')),
('http://118.844.89.244:3000', aiohttp.BasicAuth('оiVszD', '8gsdfASJzTy'))]
session = aiohttp.ClientSession()
tasks = []
responses = []
i = 0
for _ in tqdm(range(num_of_requests), desc='Fetching data...', colour='GREEN'):
if len(tasks) >= RATE_LIMIT * (len(proxies)+1):
responses += await asyncio.gather(*tasks)
for task in tasks:
task.cancel()
tasks = []
# time.sleep(TIME_TO_SLEEP)
if len(tasks) >= RATE_LIMIT:
index_of_proxy = (len(tasks) // RATE_LIMIT) - 1
tasks.append(asyncio.create_task(session.get(link, proxy=proxies[index_of_proxy][0], proxy_auth=proxies[index_of_proxy][1], headers={'User-Agent': UserAgent().random})))
else:
tasks.append(asyncio.create_task(session.get(link, headers={'User-Agent': UserAgent().random})))
i += 1
if i == num_of_requests:
print(len(tasks))
responses += await asyncio.gather(*tasks)
return [resp.status for resp in responses], session
link = "https://www.kufar.by/l/kompyuternaya-tehnika"
num_of_requests = 43 * 19
start_timestamp = time.time()
codes, session = asyncio.run(answer(link=link, num_of_requests=num_of_requests))
asyncio.run(session.close())
print(Counter(codes))
print(codes)
task_time = round(time.time() - start_timestamp, 2)
rps = round(num_of_requests / task_time, 1)
print(f"| Requests: {num_of_requests}; Total time: {task_time} s; RPS: {rps}. |\n")
