filelName.datetime.today.strftime ('%d-%b-%Y')
def process(i, file):
with open(file, "r") as f:
text = f.readline()
return i, text.replace("\n", "")
# ВАЖНО! иначе можно налететь на рекурсию
if __name__ == '__main__':
data = ( i, "../../"+row.text for i, row in tqdm(
df1[index].iterrows(),
total=df1[index].shape[0]
))
with Pool(processes=4) as pool:
for i, result in pool.imap_unordered(process, data):
df1.loc[i, "text"] = result
if i % 10000 == 0:
try:
df1.to_csv("temp.csv", index=False)
except IOError:
print("Can't save temp file")
df1.to_csv("temp.csv", index=False)asyncio.get_event_loop().create_task(my_long_task())