diff --git a/sen2chain/multi_processing.py b/sen2chain/multi_processing.py index c27d9e5d4524a8fe6fe6e78787d6ddd81a575da6..6013ca09b36da9ba0cbf526079954aa1949de1d9 100644 --- a/sen2chain/multi_processing.py +++ b/sen2chain/multi_processing.py @@ -41,10 +41,14 @@ def multi(product_copyl2asideproducts): "_OPER_", "_USER_" ) l2a_prod = L2aProduct(l2a_identifier) - timeout = time.time() + 2*60*60 + timeout = time.time() + 2 * 60 * 60 while not (l2a_prod.in_library): if time.time() > timeout: - logger.info("Timeout (2h) reached for Sen2Cor processing, killing process {}".format(proc.pid)) + logger.info( + "Timeout (2h) reached for Sen2Cor processing, killing process {}".format( + proc.pid + ) + ) os.killpg(os.getpgid(proc.pid), signal.SIGTERM) break time.sleep(5) @@ -58,13 +62,12 @@ def multi(product_copyl2asideproducts): os.killpg(os.getpgid(proc.pid), signal.SIGTERM) pass + def l2a_multiprocessing(process_list, nb_proc=4): """ """ nb_proc = max(min(len(os.sched_getaffinity(0)) - 1, nb_proc), 1) - pool = multiprocessing.Pool(nb_proc) - results = [pool.map(multi, process_list)] - pool.close() - pool.join() + with multiprocessing.Pool(nb_proc) as pool: + pool.map(multi, process_list) return True @@ -125,8 +128,7 @@ def multi_cld_ver_pro_iter_repro(l2a_ver_pro_iter_repro): def cld_version_probability_iterations_reprocessing_multiprocessing( - process_list, - nb_proc=4 + process_list, nb_proc=4 ): """ """ nb_proc = max(min(len(os.sched_getaffinity(0)) - 1, nb_proc), 1) @@ -170,7 +172,7 @@ def multi_idx(l2a_id_idx): def idx_multiprocessing( - process_list, + process_list, nb_proc=4, ): """ """