Skip to content
Snippets Groups Projects
Commit 47b07f69 authored by pascal.mouquet_ird.fr's avatar pascal.mouquet_ird.fr
Browse files

updated example script for multiprocessing using new cloudmasks

parent 83d9cd62
No related branches found
No related tags found
No related merge requests found
Pipeline #446 failed
Stage: test
Hide whitespace changes
Inline Side-by-side
examples/scripts/99_traitement_l2a_multiprocessing.py
+ 23
− 20
View file @ 47b07f69
...@@ -58,22 +58,23 @@ if l1c_process_list: ...@@ -58,22 +58,23 @@ if l1c_process_list:
if not arg: if not arg:
l2a_res = l2a_multiprocessing(l1c_process_list, nb_proc=12) l2a_res = l2a_multiprocessing(l1c_process_list, nb_proc=12)
# Traitement des L2A (clouds) # Parametrage des cloudmasks (clouds)
cm_version = "cm001" cm_version = "cm001"
probability = 1 probability = 1
iterations = 5 iterations = 5
reprocess = False cld_shad = True
cld_med_prob = True
cld_hi_prob = True
thin_cir = True
reprocess = False
# Traitement des L2A (clouds)
cld_l2a_process_list = [] cld_l2a_process_list = []
for index, row in tiles_list.iterrows(): for index, row in tiles_list.iterrows():
t = Tile(row.tile) t = Tile(row.tile)
if not reprocess: if not reprocess:
l2a_list = [p.identifier for p in t.cloudmasks_missing(cm_version = cm_version, l2a_list = [p.identifier for p in t.cloudmasks_missing().filter_dates(date_min = row.start_time, date_max = row.end_time)]
probability = probability,
iterations = iterations,
)\
.filter_dates(date_min = row.start_time, date_max = row.end_time)]
else: else:
l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = row.start_time, date_max = row.end_time)] l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = row.start_time, date_max = row.end_time)]
...@@ -82,6 +83,10 @@ for index, row in tiles_list.iterrows(): ...@@ -82,6 +83,10 @@ for index, row in tiles_list.iterrows():
cm_version, cm_version,
probability, probability,
iterations, iterations,
cld_shad,
cld_med_prob,
cld_hi_prob,
thin_cir,
reprocess, reprocess,
]) ])
logger.info("\ncld_l2a_process_list ({} files): {}".format(len(cld_l2a_process_list), cld_l2a_process_list)) logger.info("\ncld_l2a_process_list ({} files): {}".format(len(cld_l2a_process_list), cld_l2a_process_list))
...@@ -91,14 +96,10 @@ if cld_l2a_process_list: ...@@ -91,14 +96,10 @@ if cld_l2a_process_list:
if not arg: if not arg:
cld_res = cld_version_probability_iterations_reprocessing_multiprocessing(cld_l2a_process_list, nb_proc=8) cld_res = cld_version_probability_iterations_reprocessing_multiprocessing(cld_l2a_process_list, nb_proc=8)
# Traitement des L2A (indices) # Traitement des L2A (indices)
reprocess = False
nodata_clouds = True nodata_clouds = True
quicklook = False quicklook = False
cm_version = "cm001"
probability = 1
iterations = 5
indices_l2a_process_list = [] indices_l2a_process_list = []
for index, row in tiles_list.iterrows(): for index, row in tiles_list.iterrows():
...@@ -106,17 +107,13 @@ for index, row in tiles_list.iterrows(): ...@@ -106,17 +107,13 @@ for index, row in tiles_list.iterrows():
indices_list = row.indices.split("/") indices_list = row.indices.split("/")
for i in indices_list: for i in indices_list:
if not reprocess: if not reprocess:
l2a_list = [p.identifier for p in t.missing_indices(i, l2a_list = [p.identifier for p in t.missing_indices(indice = i,
nodata_clouds = nodata_clouds, nodata_clouds = nodata_clouds,
cm_version = cm_version,
probability = probability,
iterations = iterations,
).filter_dates(date_min = row.start_time, date_max = row.end_time)] ).filter_dates(date_min = row.start_time, date_max = row.end_time)]
else: else:
l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = row.start_time, date_max = row.end_time)] l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = row.start_time, date_max = row.end_time)]
for j in l2a_list: for j in l2a_list:
#~ indices_l2a_process_list.append([j, i])
indices_l2a_process_list.append([j, indices_l2a_process_list.append([j,
i, i,
reprocess, reprocess,
...@@ -124,7 +121,12 @@ for index, row in tiles_list.iterrows(): ...@@ -124,7 +121,12 @@ for index, row in tiles_list.iterrows():
quicklook, quicklook,
cm_version, cm_version,
probability, probability,
iterations]) iterations,
cld_shad,
cld_med_prob,
cld_hi_prob,
thin_cir,
])
logger.info("\nindices_l2a_process_list ({} files): {}".format(len(indices_l2a_process_list), indices_l2a_process_list)) logger.info("\nindices_l2a_process_list ({} files): {}".format(len(indices_l2a_process_list), indices_l2a_process_list))
...@@ -133,3 +135,4 @@ if indices_l2a_process_list: ...@@ -133,3 +135,4 @@ if indices_l2a_process_list:
if not arg: if not arg:
indices_res = idx_multiprocessing(indices_l2a_process_list, nb_proc=8) indices_res = idx_multiprocessing(indices_l2a_process_list, nb_proc=8)
sen2chain/tiles.py
+ 18
− 4
View file @ 47b07f69
...@@ -642,8 +642,15 @@ class Tile: ...@@ -642,8 +642,15 @@ class Tile:
""" """
if not reprocess: if not reprocess:
cld_l2a_process_list = list([p.identifier, cm_version, probability, iterations, cld_l2a_process_list = list([p.identifier,
cld_shad, cld_med_prob, cld_hi_prob, thin_cir, reprocess] \ cm_version,
probability,
iterations,
cld_shad,
cld_med_prob,
cld_hi_prob,
thin_cir,
reprocess] \
for p in self.cloudmasks_missing(cm_version = cm_version, for p in self.cloudmasks_missing(cm_version = cm_version,
probability = probability, probability = probability,
iterations = iterations, iterations = iterations,
...@@ -654,8 +661,15 @@ class Tile: ...@@ -654,8 +661,15 @@ class Tile:
)\ )\
.filter_dates(date_min = date_min, date_max = date_max)) .filter_dates(date_min = date_min, date_max = date_max))
else: else:
cld_l2a_process_list = list([p.identifier, cm_version, probability, iterations, cld_l2a_process_list = list([p.identifier,
cld_shad, cld_med_prob, cld_hi_prob, thin_cir, reprocess] \ cm_version,
probability,
iterations,
cld_shad,
cld_med_prob,
cld_hi_prob,
thin_cir,
reprocess] \
for p in self.l2a.filter_dates(date_min = date_min, date_max = date_max)) for p in self.l2a.filter_dates(date_min = date_min, date_max = date_max))
if cld_l2a_process_list: if cld_l2a_process_list:
logger.info("{} l2a products to process:".format(len(cld_l2a_process_list))) logger.info("{} l2a products to process:".format(len(cld_l2a_process_list)))
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment