From 9a4395bca4595cae6027d6a2ab62ee2db1e79561 Mon Sep 17 00:00:00 2001
From: pmouquet <pascal.mouquet@ird.fr>
Date: Thu, 25 Apr 2024 12:12:07 +0400
Subject: [PATCH] everything seems OK for removing products with Tile, Library
& Job
---
sen2chain/data/job_ini.cfg | 8 +--
sen2chain/jobs.py | 137 ++++++++++++++++++++++++++++++-------
sen2chain/library.py | 4 +-
sen2chain/tiles.py | 4 ++
4 files changed, 122 insertions(+), 31 deletions(-)
diff --git a/sen2chain/data/job_ini.cfg b/sen2chain/data/job_ini.cfg
index fa3a453..0e2b58c 100644
--- a/sen2chain/data/job_ini.cfg
+++ b/sen2chain/data/job_ini.cfg
@@ -20,15 +20,15 @@
# empty (2015-01-01 will be used) | any date | today-xx (xx nb of days before today to consider)
# date_max the last date for this task, possible values:
# empty (9999-12-31 will be used) | any date | today
-# max_clouds: max cloud cover to consider for downloading images, and computing l2a products
+# max_clouds: max cloud cover to consider for downloading images, computing l2a, cloudmask and indice products
# l1c: download l1c: True|False
# l2a: compute l2a with sen2chain: True | False
# cloudmasks: the cloudmask(s) to compute and use to mask indice(s). Possible values range from none (False) to multiple cloudmasks:
-# False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.
-# indices: False | All | NDVI/NDWIGAO/etc.
+# empty or False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.
+# indices: empty or False | All | NDVI/NDWIGAO/etc.
# remove: used to remove L1C and L2A products, considering only new products (-new) (dowloaded or produced) or the whole time serie (-all)
# and with possible filtering to keep products below specified cloud couver (-ccXX)
-# example of possible values: False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25
+# example of possible values: empty or False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25
# comments: free user comments, ie tile name, etc.
logs = False
diff --git a/sen2chain/jobs.py b/sen2chain/jobs.py
index 3ee3967..6e6037d 100644
--- a/sen2chain/jobs.py
+++ b/sen2chain/jobs.py
@@ -188,9 +188,9 @@ class Job:
("max_clouds", [100]),
("l1c", ["False"]),
("l2a", ["False"]),
- ("cloudmasks", ["False"]),
- ("indices", ["False"]),
- ("remove", ["False"]),
+ ("cloudmasks", [""]),
+ ("indices", [""]),
+ ("remove", [""]),
("comments", [""]),
]
)
@@ -304,15 +304,15 @@ class Job:
"# empty (2015-01-01 will be used) | any date | today-xx (xx nb of days before today to consider)",
"# date_max the last date for this task, possible values:",
"# empty (9999-12-31 will be used) | any date | today",
- "# max_clouds: max cloud cover to consider for downloading images, and computing l2a products ",
+ "# max_clouds: max cloud cover to consider for downloading images, computing l2a, cloudmask and indice products",
"# l1c: download l1c: True|False",
"# l2a: compute l2a with sen2chain: True | False",
"# cloudmasks: the cloudmask(s) to compute and use to mask indice(s). Possible values range from none (False) to multiple cloudmasks:",
- "# False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.",
- "# indices: False | All | NDVI/NDWIGAO/etc.",
+ "# empty or False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.",
+ "# indices: empty or False | All | NDVI/NDWIGAO/etc.",
"# remove: used to remove L1C and L2A products, considering only new products (-new) (dowloaded or produced) or the whole time serie (-all)",
"# and with possible filtering to keep products below specified cloud couver (-ccXX)",
- "# example of possible values: False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25",
+ "# example of possible values: empty or False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25",
"# comments: free user comments, ie tile name, etc.",
"",
"",
@@ -515,6 +515,7 @@ class Job:
)
self.format_cm_indices()
self.format_remove()
+ self.check_tasks()
break
except:
pass
@@ -531,11 +532,16 @@ class Job:
self.tasks.at[index, "indices"] = str(row.indices).split(
"/"
)
+ else:
+ self.tasks.at[index, "indices"] = []
if not row.cloudmasks == "False":
if not isinstance(self.tasks.at[index, "cloudmasks"] , list):
- self.tasks.at[index, "cloudmasks"] = str(row.cloudmasks).split(
- "/"
- )
+ self.tasks.at[index, "cloudmasks"] = str(row.cloudmasks).split("/")
+ else:
+ self.tasks.at[index, "cloudmasks"] = []
+ row.cloudmasks = [cm for cm in row.cloudmasks if cm != ""]
+ row.indices = [ind for ind in row.indices if ind != ""]
+
def unformat_cm_indices(self):
for index, row in self.tasks.iterrows():
@@ -553,6 +559,8 @@ class Job:
"/"
)
self.tasks.at[index, "remove"] = [b + "-new" if b == "l1c" or b == "l2a" else b for b in self.tasks.at[index, "remove"]]
+ else:
+ self.tasks.at[index, "remove"] = []
def unformat_remove(self):
for index, row in self.tasks.iterrows():
@@ -656,7 +664,73 @@ class Job:
returned_val["thin_cir"] = returned_val["thin_cir"] in ["1", 1, True, "True"]
return returned_val
+
+ def _get_ref(
+ self,
+ index,
+ row,
+ ):
+ max_cloudcover = 0
+ ref = []
+ if bool(setuptools.distutils.util.strtobool(str(row.l1c))):
+ if row.remove:
+ if bool(setuptools.distutils.util.strtobool(str(row.l2a))):
+ # row.cloudmasks = [cm for cm in row.cloudmasks if cm != ""]
+ # row.indices = [ind for ind in row.indices if ind != ""]
+ if (row.cloudmasks or row.indices):
+ for cm in row.cloudmasks:
+ ref.append(cm)
+ for ind in row.indices:
+ ref.extend([ind])
+ for cm in row.cloudmasks:
+ ref.extend([ind + "_" + cm])
+ ref = list(set(ref))
+ max_cloudcover = int(row.max_clouds)
+ else:
+ if "l2a" not in "/".join(row.remove).lower():
+ ref = "l2a"
+ max_cloudcover = int(row.max_clouds)
+ elif "l1c" not in "/".join(row.remove).lower():
+ ref = "l1c"
+ max_cloudcover = int(row.max_clouds)
+ else:
+ for rm in row.remove:
+ if "-cc" in rm:
+ cloudcover = int(rm.split("-cc")[1])
+ else:
+ cloudcover = 0
+ if cloudcover >= max_cloudcover:
+ max_cloudcover = cloudcover
+ ref = rm.split("-")[0]
+ else:
+ ref = "l1c"
+ for rm in row.remove:
+ if "-cc" in rm and rm.split("-")[0] == "l1c":
+ cloudcover = int(rm.split("-cc")[1])
+ else:
+ cloudcover = 0
+ if cloudcover >= max_cloudcover:
+ max_cloudcover = cloudcover
+ if int(row.max_clouds) > max_cloudcover:
+ logger.info(
+ "Download not optimal for line {}, downloading useless l1c:\n{}".format(
+ index,
+ self.tasks.iloc[index:index+1]
+ )
+ )
+ else:
+ ref = "l1c"
+ return ref
+
+ def check_tasks(
+ self,
+ ):
+ for index, row in self.tasks.iterrows():
+ # logger.info(row)
+ self._get_ref(index, row)
+
+
def run(
self,
# nb_proc: int = self.nb_proc,
@@ -761,26 +835,38 @@ class Job:
## Get new product list to download
download_list= []
for index, row in chunk.iterrows():
- ref = "l1c"
+ ref = self._get_ref(index, row)
+ # ref = "l1c"
if bool(setuptools.distutils.util.strtobool(str(row.l1c))):
- if not (row.remove == "False" or not row.remove):
- if "l1c" in str(row.remove).lower():
- if "l2a" not in str(row.remove).lower():
- ref = "l2a"
- else:
- ref = []
- for cm in row.cloudmasks:
- if not (row.cloudmasks == "False" or not row.cloudmasks):
- ref.append(cm)
- for ind in row.indices:
- if not (row.indices == "False" or not row.indices):
- ref.extend([ind, ind + "_" + cm])
- ref = list(set(ref))
+ # if row.remove:
+ # if not row.cloudmasks and not row.indices:
+ # max_cloudcover = 0
+ # for rm in row.remove:
+ # if "-cc" in rm:
+ # cloudcover = int(rm.split("-cc")[1])
+ # else:
+ # cloudcover = 0
+ # if cloudcover > max_cloudcover:
+ # max_cloudcover = cloudcover
+ # ref = rm.split("-")[0]
+ # if row.max_clouds > max_cloudcover_above:
+ # logger.info("Download not optimal for this line, keep downloading useless l1c:\n{}".format(row))
+ # else:
+ # if "l1c" in "/".join(row.remove).lower():
+ # if "l2a" not in "/".join(row.remove).lower():
+ # ref = "l2a"
+ # else:
+ # ref = []
+ # for cm in row.cloudmasks:
+ # ref.append(cm)
+ # for ind in row.indices:
+ # ref.extend([ind, ind + "_" + cm])
+ # ref = list(set(ref))
t = Tile(row.tile)
tile_download_list = t.get_l1c(
provider = self.provider,
- download = False,
+ download = True,
start = row.date_min,
end = row.date_max,
new = False,
@@ -1090,6 +1176,7 @@ class Job:
indices_l2a_process_list = []
for index, row in chunk.iterrows():
if not (row.indices == "False" or not row.indices):
+ logger.info("row.indices: {}".format(row.indices))
nodata_clouds = not (row.cloudmasks == "False" or not row.cloudmasks)
if nodata_clouds:
for cm in row.cloudmasks:
diff --git a/sen2chain/library.py b/sen2chain/library.py
index a5669af..45c2477 100644
--- a/sen2chain/library.py
+++ b/sen2chain/library.py
@@ -639,7 +639,7 @@ class Library:
:type jpg: bool, Default to True
"""
if tile_list is None:
- tile_list = self.l2a
+ tile_list = self.tiles_l2a
logger.info("{} L2A tile(s) to remove".format(len(tile_list)))
for tile in tile_list:
Tile(tile).remove_l2a_all(
@@ -663,7 +663,7 @@ class Library:
dates_before: str = "9999-12-31",
):
if tile_list is None:
- tile_list = self.l2a
+ tile_list = self.tiles_l2a
logger.info("{} L2A tile(s) to process".format(len(tile_list)))
for tile in tile_list:
Tile(tile).remove_l2a_filter(
diff --git a/sen2chain/tiles.py b/sen2chain/tiles.py
index 28ba419..432e866 100644
--- a/sen2chain/tiles.py
+++ b/sen2chain/tiles.py
@@ -2200,6 +2200,8 @@ class Tile:
logger.info("Removed all cloudmasks")
self._products["cloudmasks"] = ProductsList()
self._get_cloudmasks()
+ self.cloudmask_list = self._get_cloudmask_list()
+
def remove_cloudmask_filter(
self,
@@ -2235,6 +2237,8 @@ class Tile:
pass
self._products["cloudmasks"] = ProductsList()
self._get_cloudmasks()
+ self.cloudmask_list = self._get_cloudmask_list()
+
def remove_indice_all(
self,
--
GitLab