Skip to content
Snippets Groups Projects
Commit 9a4395bc authored by pascal.mouquet_ird.fr's avatar pascal.mouquet_ird.fr
Browse files

everything seems OK for removing products with Tile, Library & Job

parent 1738a442
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
sen2chain/data/job_ini.cfg
+ 4
− 4
View file @ 9a4395bc
......@@ -20,15 +20,15 @@
# empty (2015-01-01 will be used) | any date | today-xx (xx nb of days before today to consider)
# date_max the last date for this task, possible values:
# empty (9999-12-31 will be used) | any date | today
# max_clouds: max cloud cover to consider for downloading images, and computing l2a products
# max_clouds: max cloud cover to consider for downloading images, computing l2a, cloudmask and indice products
# l1c: download l1c: True|False
# l2a: compute l2a with sen2chain: True | False
# cloudmasks: the cloudmask(s) to compute and use to mask indice(s). Possible values range from none (False) to multiple cloudmasks:
# False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.
# indices: False | All | NDVI/NDWIGAO/etc.
# empty or False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.
# indices: empty or False | All | NDVI/NDWIGAO/etc.
# remove: used to remove L1C and L2A products, considering only new products (-new) (dowloaded or produced) or the whole time serie (-all)
# and with possible filtering to keep products below specified cloud couver (-ccXX)
# example of possible values: False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25
# example of possible values: empty or False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25
# comments: free user comments, ie tile name, etc.
logs = False
......
sen2chain/jobs.py
+ 112
− 25
View file @ 9a4395bc
......@@ -188,9 +188,9 @@ class Job:
("max_clouds", [100]),
("l1c", ["False"]),
("l2a", ["False"]),
("cloudmasks", ["False"]),
("indices", ["False"]),
("remove", ["False"]),
("cloudmasks", [""]),
("indices", [""]),
("remove", [""]),
("comments", [""]),
]
)
......@@ -304,15 +304,15 @@ class Job:
"# empty (2015-01-01 will be used) | any date | today-xx (xx nb of days before today to consider)",
"# date_max the last date for this task, possible values:",
"# empty (9999-12-31 will be used) | any date | today",
"# max_clouds: max cloud cover to consider for downloading images, and computing l2a products ",
"# max_clouds: max cloud cover to consider for downloading images, computing l2a, cloudmask and indice products",
"# l1c: download l1c: True|False",
"# l2a: compute l2a with sen2chain: True | False",
"# cloudmasks: the cloudmask(s) to compute and use to mask indice(s). Possible values range from none (False) to multiple cloudmasks:",
"# False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.",
"# indices: False | All | NDVI/NDWIGAO/etc.",
"# empty or False | CM001/CM002/CM003-PRB1-ITER5/CM004-CSH1-CMP1-CHP1-TCI1-ITER0/etc.",
"# indices: empty or False | All | NDVI/NDWIGAO/etc.",
"# remove: used to remove L1C and L2A products, considering only new products (-new) (dowloaded or produced) or the whole time serie (-all)",
"# and with possible filtering to keep products below specified cloud couver (-ccXX)",
"# example of possible values: False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25",
"# example of possible values: empty or False | l1c-new-cc80 | l1c-all | l2a-new | l1c-all/l2a-new-cc25",
"# comments: free user comments, ie tile name, etc.",
"",
"",
......@@ -515,6 +515,7 @@ class Job:
)
self.format_cm_indices()
self.format_remove()
self.check_tasks()
break
except:
pass
......@@ -531,11 +532,16 @@ class Job:
self.tasks.at[index, "indices"] = str(row.indices).split(
"/"
)
else:
self.tasks.at[index, "indices"] = []
if not row.cloudmasks == "False":
if not isinstance(self.tasks.at[index, "cloudmasks"] , list):
self.tasks.at[index, "cloudmasks"] = str(row.cloudmasks).split(
"/"
)
self.tasks.at[index, "cloudmasks"] = str(row.cloudmasks).split("/")
else:
self.tasks.at[index, "cloudmasks"] = []
row.cloudmasks = [cm for cm in row.cloudmasks if cm != ""]
row.indices = [ind for ind in row.indices if ind != ""]
def unformat_cm_indices(self):
for index, row in self.tasks.iterrows():
......@@ -553,6 +559,8 @@ class Job:
"/"
)
self.tasks.at[index, "remove"] = [b + "-new" if b == "l1c" or b == "l2a" else b for b in self.tasks.at[index, "remove"]]
else:
self.tasks.at[index, "remove"] = []
def unformat_remove(self):
for index, row in self.tasks.iterrows():
......@@ -656,7 +664,73 @@ class Job:
returned_val["thin_cir"] = returned_val["thin_cir"] in ["1", 1, True, "True"]
return returned_val
def _get_ref(
self,
index,
row,
):
max_cloudcover = 0
ref = []
if bool(setuptools.distutils.util.strtobool(str(row.l1c))):
if row.remove:
if bool(setuptools.distutils.util.strtobool(str(row.l2a))):
# row.cloudmasks = [cm for cm in row.cloudmasks if cm != ""]
# row.indices = [ind for ind in row.indices if ind != ""]
if (row.cloudmasks or row.indices):
for cm in row.cloudmasks:
ref.append(cm)
for ind in row.indices:
ref.extend([ind])
for cm in row.cloudmasks:
ref.extend([ind + "_" + cm])
ref = list(set(ref))
max_cloudcover = int(row.max_clouds)
else:
if "l2a" not in "/".join(row.remove).lower():
ref = "l2a"
max_cloudcover = int(row.max_clouds)
elif "l1c" not in "/".join(row.remove).lower():
ref = "l1c"
max_cloudcover = int(row.max_clouds)
else:
for rm in row.remove:
if "-cc" in rm:
cloudcover = int(rm.split("-cc")[1])
else:
cloudcover = 0
if cloudcover >= max_cloudcover:
max_cloudcover = cloudcover
ref = rm.split("-")[0]
else:
ref = "l1c"
for rm in row.remove:
if "-cc" in rm and rm.split("-")[0] == "l1c":
cloudcover = int(rm.split("-cc")[1])
else:
cloudcover = 0
if cloudcover >= max_cloudcover:
max_cloudcover = cloudcover
if int(row.max_clouds) > max_cloudcover:
logger.info(
"Download not optimal for line {}, downloading useless l1c:\n{}".format(
index,
self.tasks.iloc[index:index+1]
)
)
else:
ref = "l1c"
return ref
def check_tasks(
self,
):
for index, row in self.tasks.iterrows():
# logger.info(row)
self._get_ref(index, row)
def run(
self,
# nb_proc: int = self.nb_proc,
......@@ -761,26 +835,38 @@ class Job:
## Get new product list to download
download_list= []
for index, row in chunk.iterrows():
ref = "l1c"
ref = self._get_ref(index, row)
# ref = "l1c"
if bool(setuptools.distutils.util.strtobool(str(row.l1c))):
if not (row.remove == "False" or not row.remove):
if "l1c" in str(row.remove).lower():
if "l2a" not in str(row.remove).lower():
ref = "l2a"
else:
ref = []
for cm in row.cloudmasks:
if not (row.cloudmasks == "False" or not row.cloudmasks):
ref.append(cm)
for ind in row.indices:
if not (row.indices == "False" or not row.indices):
ref.extend([ind, ind + "_" + cm])
ref = list(set(ref))
# if row.remove:
# if not row.cloudmasks and not row.indices:
# max_cloudcover = 0
# for rm in row.remove:
# if "-cc" in rm:
# cloudcover = int(rm.split("-cc")[1])
# else:
# cloudcover = 0
# if cloudcover > max_cloudcover:
# max_cloudcover = cloudcover
# ref = rm.split("-")[0]
# if row.max_clouds > max_cloudcover_above:
# logger.info("Download not optimal for this line, keep downloading useless l1c:\n{}".format(row))
# else:
# if "l1c" in "/".join(row.remove).lower():
# if "l2a" not in "/".join(row.remove).lower():
# ref = "l2a"
# else:
# ref = []
# for cm in row.cloudmasks:
# ref.append(cm)
# for ind in row.indices:
# ref.extend([ind, ind + "_" + cm])
# ref = list(set(ref))
t = Tile(row.tile)
tile_download_list = t.get_l1c(
provider = self.provider,
download = False,
download = True,
start = row.date_min,
end = row.date_max,
new = False,
......@@ -1090,6 +1176,7 @@ class Job:
indices_l2a_process_list = []
for index, row in chunk.iterrows():
if not (row.indices == "False" or not row.indices):
logger.info("row.indices: {}".format(row.indices))
nodata_clouds = not (row.cloudmasks == "False" or not row.cloudmasks)
if nodata_clouds:
for cm in row.cloudmasks:
......
sen2chain/library.py
+ 2
− 2
View file @ 9a4395bc
......@@ -639,7 +639,7 @@ class Library:
:type jpg: bool, Default to True
"""
if tile_list is None:
tile_list = self.l2a
tile_list = self.tiles_l2a
logger.info("{} L2A tile(s) to remove".format(len(tile_list)))
for tile in tile_list:
Tile(tile).remove_l2a_all(
......@@ -663,7 +663,7 @@ class Library:
dates_before: str = "9999-12-31",
):
if tile_list is None:
tile_list = self.l2a
tile_list = self.tiles_l2a
logger.info("{} L2A tile(s) to process".format(len(tile_list)))
for tile in tile_list:
Tile(tile).remove_l2a_filter(
......
sen2chain/tiles.py
+ 4
− 0
View file @ 9a4395bc
......@@ -2200,6 +2200,8 @@ class Tile:
logger.info("Removed all cloudmasks")
self._products["cloudmasks"] = ProductsList()
self._get_cloudmasks()
self.cloudmask_list = self._get_cloudmask_list()
def remove_cloudmask_filter(
self,
......@@ -2235,6 +2237,8 @@ class Tile:
pass
self._products["cloudmasks"] = ProductsList()
self._get_cloudmasks()
self.cloudmask_list = self._get_cloudmask_list()
def remove_indice_all(
self,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment