from domainlist import site_pages, real_site,site_pages_mini, site_pages_react, site_pages_all,site_pages_random_3000,site_pages_random_1000, site_pages_random_2000, list_5000
from memory_profiler import memory_usage
import random
import aiohttp
import asyncio
import time
async def fetch(url, session):
headers = {
"Accept":"application/json, text/plain, */*",
"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
"Accept-Language":"zh-CN,zh;q=0.9",
"accept-encoding":"gzip, deflate"}
try:
async with session.get(url, timeout=0.1, headers=headers) as response:
date = response.headers.get("DATE")
code_status = response.history[0].status if response.history else response.status
print('%s -> Status Code: %s' % (url, code_status))
return await response.read()
except Exception as e:
print("{} : exeption {}/ class {}".format(url, e, e.__class__))
pass
async def bound_fetch(sem, url, session):
async with sem:
await fetch(url, session)
async def run():
tasks = []
check_site = real_site
sem = asyncio.Semaphore(1000)
async with aiohttp.ClientSession() as session:
for url in check_site:
task = asyncio.ensure_future(bound_fetch(sem, url, session))
tasks.append(task)
responses = asyncio.gather(*tasks)
await responses
loop = asyncio.get_event_loop()
global_start_time = time.time()
future = asyncio.ensure_future(run())
loop.run_until_complete(future)
print("--- %s seconds ---" % (time.time() - global_start_time))
sem = asyncio.Semaphore(1000)
async with aiohttp.ClientSession() as session:
for url in check_site:
task = asyncio.ensure_future(bound_fetch(sem, url, session))
tasks.append(task)
responses = asyncio.gather(*tasks)
await responses
loop = asyncio.get_event_loop()
global_start_time = time.time()
future = asyncio.ensure_future(run())
loop.run_until_complete(future)
http://yandex.ru : exeption / class <class 'concurrent.futures._base.TimeoutError'>
http://yandex.ru -> Status Code: 302
Wed, 04 Sep 2019 10:23:46 GMT:https://yandex.ru/ with delay
composer require symfony/orm-pack
Using version ^1.0 for symfony/orm-pack
./composer.json has been updated
Loading composer repositories with package information
Updating dependencies (including require-dev)
Package operations: 6 installs, 0 updates, 0 removals
- Installing zendframework/zend-eventmanager (3.2.1): Downloading (connecting.Downloading (100%)
- Installing zendframework/zend-code (3.3.1): Downloading (100%)
- Installing ocramius/proxy-manager (2.2.2): Downloading (100%)
- Installing doctrine/migrations (2.1.1): Downloading (100%)
- Installing doctrine/doctrine-migrations-bundle (v2.0.0): Downloading (connecDownloading (100%)
- Installing symfony/orm-pack (v1.0.6): Downloading (100%)
zendframework/zend-eventmanager suggests installing container-interop/container-interop (^1.1.0, to use the lazy listeners feature)
zendframework/zend-eventmanager suggests installing zendframework/zend-stdlib (^2.7.3 || ^3.0, to use the FilterChain feature)
zendframework/zend-code suggests installing zendframework/zend-stdlib (Zend\Stdlib component)
ocramius/proxy-manager suggests installing ocramius/generated-hydrator (To have very fast object to array to object conversion for ghost objects)
ocramius/proxy-manager suggests installing zendframework/zend-xmlrpc (To have the XmlRpc adapter (Remote Object feature))
ocramius/proxy-manager suggests installing zendframework/zend-json (To have the JsonRpc adapter (Remote Object feature))
ocramius/proxy-manager suggests installing zendframework/zend-soap (To have the Soap adapter (Remote Object feature))
Package symfony/assetic-bundle is abandoned, you should avoid using it. Use symfony/webpack-encore-pack instead.
Writing lock file
Generating autoload files
ocramius/package-versions: Generating version class...
ocramius/package-versions: ...done generating version class
> php bin/console assetic:dump .
PHP Fatal error: Uncaught SpoonDatabaseException: A database connection could not be established. in /opt/lampp/htdocs/atrib/vendor/spoon/library/spoon/database/database.php:178
Stack trace:
#0 /opt/lampp/htdocs/atrib/vendor/spoon/library/spoon/database/database.php(264): SpoonDatabase->connect()
#1 /opt/lampp/htdocs/atrib/var/cache/dev/ContainerIp3hdlx/appDevDebugProjectContainer.php(842): SpoonDatabase->execute('SET CHARACTER S...', Array)
#2 /opt/lampp/htdocs/atrib/var/cache/dev/ContainerIp3hdlx/appDevDebugProjectContainer.php(1008): ContainerIp3hdlx\appDevDebugProjectContainer->getDatabaseService()
#3 /opt/lampp/htdocs/atrib/var/cache/dev/ContainerIp3hdlx/appDevDebugProjectContainer.php(1068): ContainerIp3hdlx\appDevDebugProjectContainer->getFork_SettingsService()
#4 /opt/lampp/htdocs/atrib/var/cache/dev/ContainerIp3hdlx/appDevDebugProjectContainer.php(1816): ContainerIp3hdlx\appDevDebugProjectContainer->getMediaLibrary_Manager_FileService()
#5 /opt/lampp/htdocs/atrib/var/cache/dev/ContainerIp3hdlx/appDevDebugProjectConta in /opt/lampp/htdocs/atrib/vendor/spoon/library/spoon/database/database.php on line 178
Script php bin/console assetic:dump . handling the post-update-cmd event returned with error code 255
По 3-6 секунды, хотя открываю сайт, смотрю "инспектор", там за 1-2 секунды.
Ну ладно, спасибо.
Я просто думал я может в документации что пропустил.