import asyncio
import requests
from mysql import MySql
import concurrent.futures
def get_data(id):
mysql = MySql()
while True:
resp = requests.get("http://site.ru/path?id=" + id)
mysql.insert_data(resp)
return 1
async def main():
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
ids = [1,2,3,4,5, ... , N];
loop = asyncio.get_event_loop()
futures = [
loop.run_in_executor(
executor,
get_data,
id
)
for id in ids
]
for response in await asyncio.gather(*futures):
pass
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
values = [[a1,a2,...], [b1, b2,...], ....]
query = "INSERT INTO ads_irr_ru_content (col1, col2, .... ,col5) VALUES " +
",".join("(%s,%s,%s,%s,%s)" for _ in values)
flattened_values = [item for sublist in values for item in sublist]
cur.execute(query, flattened_values)