def multiprocess_list(mp_type, work_list, settings, logger):
pool = multiprocessing.Pool(processes=settings['max_concurrent_processes'])
num_jobs = len(work_list)
if mp_type == "sql":
results = pool.imap_unordered(run_sql_multiprocessing, [[w, settings] for w in work_list])
else:
results = pool.imap_unordered(run_command_line, work_list)
pool.close()
pool.join()
result_list = list(results)
num_results = len(result_list)
if num_jobs > num_results:
logger.warning("\t- A MULTIPROCESSING PROCESS FAILED WITHOUT AN ERROR\nACTION: Check the record counts")
for result in result_list:
if result != "SUCCESS":
logger.info(result)
评论列表
文章目录