Предлагаю вам свой вариант, через очередь задач:
from Queue import Queue
from threading import Thread
def scrape(link):
conn = sqlite3.connect('db.sqlite3')
curs = conn.cursor()
# insert link, close connection
class Worker(Thread):
def __init__(self, tasks):
super(Worker, self).__init__()
self.tasks = tasks
self.daemon = True
def run(self):
while True:
link = self.tasks.get()
try:
scrape(link)
finally:
self.tasks.task_done()
if __name__ == '__main__':
# максимальное количество одновременных потоков
capacity = 0 # infinite
queue = Queue(capacity)
workers = 3
for _ in range(workers):
Worker(queue).start()
for link in select.fetchall():
queue.put(link)
queue.join()
print 'Done'
