From 571050da6608686a9eaf32ef087d2f91d027c1af Mon Sep 17 00:00:00 2001
From: Impact <pascal.mouquet@ird.fr>
Date: Tue, 17 Nov 2020 11:01:44 +0400
Subject: [PATCH] 3 new Library functions (compute_l2a, compute_cloudmasks,
compute_indices) to multi-process a list of tiles
---
sen2chain/library.py | 88 +++++++++++++++++++++++++++++++++++++++-----
1 file changed, 79 insertions(+), 9 deletions(-)
diff --git a/sen2chain/library.py b/sen2chain/library.py
index 1763980..42c7448 100644
--- a/sen2chain/library.py
+++ b/sen2chain/library.py
@@ -8,6 +8,7 @@ import os
import shutil
import logging
import pathlib
+from itertools import chain
from pathlib import Path
# type annotations
@@ -17,6 +18,7 @@ from .config import Config, SHARED_DATA
from .products import L1cProduct
from .tiles import Tile
from .utils import human_size, human_size_decimal
+from .multi_processing import l2a_multiprocessing, cld_version_probability_iterations_reprocessing_multiprocessing, idx_multiprocessing
s2_tiles_index = SHARED_DATA.get("tiles_index")
@@ -207,24 +209,92 @@ class Library:
for tile in set([val for sublist in [k for k in [getattr(self,t) for t in self.__dict__]] for val in sublist]):
Tile(tile).init_md()
- ####################### A FAIRE
def compute_l2a(self,
tile_list: list = [],
+ date_min: str = None,
+ date_max: str = None,
+ nb_proc: int = 4,
):
- toto = 12
+ l1c_process_list = []
+ for tile in tile_list:
+ t = Tile(tile)
+ l1c_process_list.append(list(p.identifier for p in t.l2a_missings.filter_dates(date_min = date_min, date_max = date_max)))
+ l1c_process_list = list(chain.from_iterable(l1c_process_list))
+ logger.info("l1c_process_list ({} files): \n{}".format(len(l1c_process_list), l1c_process_list))
+
+ if l1c_process_list:
+ l2a_res = l2a_multiprocessing(l1c_process_list, nb_proc=nb_proc)
- ####################### A FAIRE
def compute_cloudmasks(self,
tile_list: list = [],
+ cm_version: str = "cm001",
+ probability: int = 1,
+ iterations: int = 5,
+ reprocess: bool = False,
+ date_min: str = None,
+ date_max: str = None,
+ nb_proc: int = 4,
):
- toto = 15
-
- ####################### A FAIRE
+ cld_l2a_process_list = []
+ for tile in tile_list:
+ t = Tile(tile)
+ if not reprocess:
+ l2a_list = [p.identifier for p in t.cloudmasks_missing(cm_version = cm_version,
+ probability = probability,
+ iterations = iterations,
+ )\
+ .filter_dates(date_min = date_min, date_max = date_max)]
+ else:
+ l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = date_min, date_max = date_max)]
+ for j in l2a_list:
+ cld_l2a_process_list.append([j,
+ cm_version,
+ probability,
+ iterations,
+ reprocess,
+ ])
+ logger.info("cld_l2a_process_list ({} files): \n{}".format(len(cld_l2a_process_list), cld_l2a_process_list))
+ if cld_l2a_process_list:
+ cld_res = cld_version_probability_iterations_reprocessing_multiprocessing(cld_l2a_process_list, nb_proc=8)
+
def compute_indices(self,
tile_list: list = [],
- ):
- toto = 20
-
+ indice_list: list = [],
+ reprocess: bool = False,
+ nodata_clouds: bool = True,
+ quicklook: bool = False,
+ cm_version: list = "cm001",
+ probability: int = 1,
+ iterations: int = 5,
+ date_min: str = None,
+ date_max: str = None,
+ nb_proc: int = 4):
+ indices_l2a_process_list = []
+ for tile in tile_list:
+ t = Tile(tile)
+ for i in indice_list:
+ if not reprocess:
+ l2a_list = [p.identifier for p in t.missing_indices(i,
+ nodata_clouds = nodata_clouds,
+ cm_version = cm_version,
+ probability = probability,
+ iterations = iterations,
+ ).filter_dates(date_min = date_min, date_max = date_max)]
+ else:
+ l2a_list = [p.identifier for p in t.l2a.filter_dates(date_min = date_min, date_max = date_max)]
+
+ for j in l2a_list:
+ indices_l2a_process_list.append([j,
+ i,
+ reprocess,
+ nodata_clouds,
+ quicklook,
+ cm_version,
+ probability,
+ iterations])
+ logger.info("indices_l2a_process_list ({} files): \n{}".format(len(indices_l2a_process_list), indices_l2a_process_list))
+ if indices_l2a_process_list:
+ indices_res = idx_multiprocessing(indices_l2a_process_list, nb_proc=8)
class TempContainer:
--
GitLab