3 new Library functions (compute_l2a, compute_cloudmasks, compute_indices) to...

3 new Library functions (compute_l2a, compute_cloudmasks, compute_indices) to multi-process a list of tiles

sen2chain/library.py

@@ -8,6 +8,7 @@ import os
 import shutil
 import logging
 import pathlib
+from itertools import chain
 
 from pathlib import Path
 # type annotations
@@ -17,6 +18,7 @@ from .config import Config, SHARED_DATA
 from .products import L1cProduct
 from .tiles import Tile
 from .utils import human_size, human_size_decimal
+from .multi_processing import l2a_multiprocessing, cld_version_probability_iterations_reprocessing_multiprocessing, idx_multiprocessing
 
 s2_tiles_index = SHARED_DATA.get("tiles_index")
 
@@ -207,24 +209,92 @@ class Library:
         for tile in set([val for sublist in [k for k in [getattr(self,t) for t in self.__dict__]] for val in sublist]):
            Tile(tile).init_md()
     
-    ####################### A FAIRE
     def compute_l2a(self, 
                     tile_list: list = [],
+                    date_min: str = None,
+                    date_max: str = None,
+                    nb_proc: int = 4, 
                     ):
-        toto = 12
+        l1c_process_list =  []
+        for tile in tile_list:
+            t = Tile(tile)
+            l1c_process_list.append(list(p.identifier for p in t.l2a_missings.filter_dates(date_min = date_min, date_max = date_max)))
+        l1c_process_list = list(chain.from_iterable(l1c_process_list))
+        logger.info("l1c_process_list ({} files): \n{}".format(len(l1c_process_list), l1c_process_list))
+
+        if l1c_process_list:
+            l2a_res = l2a_multiprocessing(l1c_process_list, nb_proc=nb_proc)
     
-    ####################### A FAIRE
     def compute_cloudmasks(self, 
                     tile_list: list = [],
+                    cm_version: str = "cm001",
+                    probability: int = 1,
+                    iterations: int = 5,
+                    reprocess: bool = False,
+                    date_min: str = None,
+                    date_max: str = None,
+                    nb_proc: int = 4,
                     ):
-        toto = 15
-        
-    ####################### A FAIRE
+        cld_l2a_process_list =  []
+        for tile in tile_list:
+            t = Tile(tile)
+            if not reprocess:
+                l2a_list = [p.identifier for p in t.cloudmasks_missing(cm_version = cm_version,
+                                                                       probability = probability,
+                                                                       iterations = iterations,
+                                                                       )\
+                                                                       .filter_dates(date_min = date_min, date_max = date_max)]
+            else:
+                l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = date_min, date_max = date_max)]
+            for j in l2a_list:
+                cld_l2a_process_list.append([j, 
+                                             cm_version,
+                                             probability,
+                                             iterations,
+                                             reprocess,
+                                             ])
+        logger.info("cld_l2a_process_list ({} files): \n{}".format(len(cld_l2a_process_list), cld_l2a_process_list))
+        if cld_l2a_process_list:
+            cld_res = cld_version_probability_iterations_reprocessing_multiprocessing(cld_l2a_process_list, nb_proc=8)
+    
     def compute_indices(self, 
                     tile_list: list = [],
-                    ):
-        toto = 20
-
+                    indice_list: list = [],                        
+                    reprocess: bool = False,
+                    nodata_clouds: bool = True,
+                    quicklook: bool = False,
+                    cm_version: list = "cm001",
+                    probability: int = 1,
+                    iterations: int = 5,
+                    date_min: str = None,
+                    date_max: str = None,
+                    nb_proc: int = 4):
+        indices_l2a_process_list =  []
+        for tile in tile_list:
+            t = Tile(tile)
+            for i in indice_list:
+                if not reprocess:
+                    l2a_list = [p.identifier for p in t.missing_indices(i,
+                                                                        nodata_clouds = nodata_clouds,
+                                                                        cm_version = cm_version,
+                                                                        probability = probability, 
+                                                                        iterations = iterations,
+                                                                        ).filter_dates(date_min = date_min, date_max = date_max)]
+                else:
+                    l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = date_min, date_max = date_max)]
+                
+                for j in l2a_list:
+                    indices_l2a_process_list.append([j, 
+                                                     i,
+                                                     reprocess,
+                                                     nodata_clouds,
+                                                     quicklook,
+                                                     cm_version,
+                                                     probability,
+                                                     iterations])
+        logger.info("indices_l2a_process_list ({} files): \n{}".format(len(indices_l2a_process_list), indices_l2a_process_list))
+        if indices_l2a_process_list:
+            indices_res = idx_multiprocessing(indices_l2a_process_list, nb_proc=8)
 
 
 class TempContainer: