Проблемы с соединением при многопоточной загрузке файла на питоне, как решить?

Question

astrotrain @astrotrain

Проблемы с соединением при многопоточной загрузке файла на питоне, как решить?

Вот полный код моего загрузчика:

import threading
import queue
import time
import re
import pprint
import ftplib


list_unvalid = []
list_valid = []

mylock = threading.RLock()

class StatusControl():

    def __init__(self, data):
        self.data = data
        self.data_count = len(data)
        self.total_processed = 0
        self.percent_diff = 5
        self.last_percent = 0      
        self.total_valid = 0  

    def addTotal(self):
        self.total_processed += 1

    def getDataLen(self):
        return self.data_count

    def gerCurrPercent(self):
        return self.last_percent

    def checkPercentDiff(self):
        current_percent =  int(( self.total_processed / self.data_count ) * 100)
        if current_percent - self.last_percent >= self.percent_diff:
            self.last_percent = current_percent
            return True
        return False

    def addValid(self):
        self.total_valid += 1

def worker_function(input_queue, output_queue, status_control):
    global list_unvalid, list_valid

    while True:
        try:
            data = input_queue.get_nowait()
        except queue.Empty:
            break
        #print("checking " + str(data))
        try:

            #line = 'username:password@my.domain.com/dir/aaa/aaa domain.com'
            data = data.replace('ftp://', '')
            login_data = re.findall('(.+?):(.+?)@(.+?)(/.[^\s]+)\s*(.*)', data, flags=re.IGNORECASE)
            print(login_data)
            login = login_data[0][0]
            password = login_data[0][1]
            host = login_data[0][2]
            directory = login_data[0][3]            
            try:
                url = login_data[0][4]
            except IndexError:
                url = 'http://' + host            
            ftp_connection = ftplib.FTP(host, login, password)
            #ftp_connection.login()
            ftp_connection.cwd(directory)
            dirs = ftp_connection.nlst()
            print(dirs)
            ftp_connection.close()
            status_control.addTotal()
            mylock.acquire()
            diff = status_control.checkPercentDiff()
            if diff == True:
                print(status_control.gerCurrPercent() + '%')
            mylock.release()       
            output_queue.put_nowait(data)

        except Exception as e:
            print(str(e))

        input_queue.task_done()


def main():
    start_t = time.time()

    f = open('ftp.txt', 'r')
    lines = f.readlines()
    lines = [line.strip() for line in lines]


    in_queue = queue.Queue()
    out_queue = queue.Queue()

    for line in lines:
        if(line.strip() == ''):
            continue
        data = line
        in_queue.put_nowait(data)

    status_control = StatusControl(lines)

    worker_threads = 100
    threads = []
    for thr in range(worker_threads):
        thread = threading.Thread(target=worker_function, args=(in_queue, out_queue, status_control))
        thread.daemon = True
        threads.append(thread)
        thread.start()

    for thread in threads:
        thread.join()

    out_list = []


    while True:
        try:
            info = out_queue.get_nowait()           
            out_list.append(info)
        except queue.Empty:
            break
    print(out_list)
    total_time = time.time() - start_t
    print('Total time ' + str(total_time) + ' seconds')

if __name__ == '__main__':
    main()

Когда в файле несколько фтп, тогда все более-менее нормально, но как-то их становится 5 или больше вся консоль засыпана ошибками: