From 3388074666b16c76d60c9d3ad698d8acb8f5af04 Mon Sep 17 00:00:00 2001
From: pmouquet <pascal.mouquet@ird.fr>
Date: Mon, 27 Mar 2023 11:44:47 +0400
Subject: [PATCH] added ref to Job for downloading l1c
---
sen2chain/download_eodag.py | 32 ++++++++++-----------------
sen2chain/jobs.py | 44 +++++++++++++++++++++++++++----------
sen2chain/products.py | 11 +++++++++-
sen2chain/tiles.py | 2 +-
sen2chain/utils.py | 18 +++++++--------
5 files changed, 64 insertions(+), 43 deletions(-)
diff --git a/sen2chain/download_eodag.py b/sen2chain/download_eodag.py
index d83567c..43af554 100644
--- a/sen2chain/download_eodag.py
+++ b/sen2chain/download_eodag.py
@@ -74,7 +74,7 @@ class S2cEodag:
productType: str = "L1C",
start: str = "2015-01-01",
end: str = "9999-12-31",
- ref: str = "l1c",
+ ref = "l1c",
min_cloudcover: int = 0,
max_cloudcover: int = 100,
):
@@ -100,10 +100,10 @@ class S2cEodag:
)
logging.disable(logging.NOTSET)
- if not isinstance(ref, list):
- ref_string = ref
- else:
- ref_string = "specific cloudmask(s) and / or indice(s) product(s)"
+ # if not isinstance(ref, list):
+ # ref_string = ref
+ # else:
+ # ref_string = "specific cloudmask(s) and / or indice(s) product(s)"
logger.info("Searching online products, ref: {}".format(ref))
fitered = self.products[:]
@@ -243,21 +243,13 @@ class S2cEodag:
p.properties["storageStatus"]
)
)
- elif not isinstance(ref, str):
- fp = FamilyProduct(p.properties["title"])
- all_conditions = []
+ else:
if not isinstance(ref, list):
ref = [ref]
+ fp = FamilyProduct(p.properties["title"])
+ all_conditions = []
for r in ref:
- if fp.cloudmasks or fp.indices:
- if "indice_string" in r.keys():
- if fp.indices:
- all_conditions.append(r["indice_string"] in [ind["indice_string"] for ind in fp.indices])
- elif "cm_string" in r.keys():
- if fp.cloudmasks:
- all_conditions.append(r["cm_string"] in [cm["cm_string"] for cm in fp.cloudmasks])
- else:
- all_conditions.append(False)
+ all_conditions.append(r in fp.cm_ind_string_list)
if all(all_conditions):
logger.info(
"{} - all ref PRESENT - filtering".format(
@@ -268,7 +260,7 @@ class S2cEodag:
else:
if l2a_presence == "PRESENT":
logger.info(
- "{} - l2a PRESENT - some ref ABSENT - filtering".format(
+ "{} - some ref ABSENT - l2a PRESENT - filtering".format(
p.properties["title"],
)
)
@@ -276,7 +268,7 @@ class S2cEodag:
else:
if l1c_presence == "PRESENT":
logger.info(
- "{} - l1c PRESENT - some ref ABSENT - filtering".format(
+ "{} - some ref ABSENT - l1c PRESENT - filtering".format(
p.properties["title"],
l1c_presence,
p.properties["storageStatus"]
@@ -285,7 +277,7 @@ class S2cEodag:
self.products.remove(p)
else:
logger.info(
- "{} - l1c ABSENT, l2a ABSENT, some ref ABSENT - remote {}".format(
+ "{} - some ref ABSENT - l2a ABSENT - l1c ABSENT - downloading - remote {}".format(
p.properties["title"],
p.properties["storageStatus"]
)
diff --git a/sen2chain/jobs.py b/sen2chain/jobs.py
index 359d254..58a7814 100644
--- a/sen2chain/jobs.py
+++ b/sen2chain/jobs.py
@@ -629,8 +629,15 @@ class Job:
if "l2a" not in str(row.remove).lower():
ref = "l2a"
else:
-
-
+ ref = []
+ for cm in row.cloudmasks:
+ if not (row.cloudmasks == "False" or not row.cloudmasks):
+ ref.append(cm)
+ for ind in row.indices:
+ if not (row.indices == "False" or not row.indices):
+ ref.extend([ind, ind + "_" + cm])
+ ref = list(set(ref))
+
t = Tile(row.tile)
tile_download_list = t.get_l1c(
provider = self.provider,
@@ -758,14 +765,14 @@ class Job:
# Remove downloaded L1C
for index, row in tasks.iterrows():
if "l1c" in str(row.remove).lower():
+ logger.info("Removing downloaded l1c products for tile: {}".format(row.tile))
+ if self.logs:
+ f.write("{}\nRemoving downloaded l1c products for tile: {}\n\n".format(datetime.datetime.now(), row.tile))
+ f.flush()
t = Tile(row.tile)
prodlist = [p for p in l1c_process_list if row.tile in p]
t.remove_l1c(prodlist)
- logger.info("Removing downloaded l1c products: {}".format(prodlist))
- if self.logs:
- f.write("{}\nRemoving downloaded l1c products: {}\n\n".format(datetime.datetime.now(), prodlist))
- f.flush()
-
+
# Comuting cloudmasks (from L2A)
logger.info("Computing cloudmasks")
if self.logs:
@@ -957,14 +964,27 @@ class Job:
if "l2a" in str(row.remove).lower():
t = Tile(row.tile)
prodlist = [p for p in l2a_remove_list if row.tile in p]
- t.remove_l2a(prodlist)
- logger.info("Removing {} produced l2a product(s): {}".format(len(prodlist), prodlist))
+ logger.info(
+ "Removing {} produced l2a for tile {}".format(
+ row.tile,
+ len(prodlist),
+ )
+ )
if self.logs:
- f.write("Removing {} produced l2a products: {}\n".format(len(prodlist)))
+ f.write(
+ "{}\nRemoving {} produced l2a for tile {}\n".format(
+ datetime.datetime.now(),
+ len(prodlist),
+ row.tile,
+ )
+ )
for l2a in prodlist:
- f.write("{}\n\n".format(l2a))
- f.flush()
+ f.write("{}\n".format(l2a))
+ f.write("\n")
+ f.flush()
+ t.remove_l2a(prodlist)
+
# Cleaning after
if clean_after:
logger.info("Cleaning Tiles")
diff --git a/sen2chain/products.py b/sen2chain/products.py
index ffc35a3..98e28bb 100755
--- a/sen2chain/products.py
+++ b/sen2chain/products.py
@@ -1770,6 +1770,7 @@ class FamilyProduct(dict):
self.l2a_id = self.get_l2a()
self.cloudmasks = self.get_cloudmasks()
self.indices = self.get_indices()
+ self.cm_ind_string_list = self.get_cm_ind_string_list()
else:
logger.info("Invalid identifier {}".format(identifier))
for key, val in self.__dict__.items():
@@ -1845,4 +1846,12 @@ class FamilyProduct(dict):
indices.append(indice_dict)
except:
pass
- return indices
+ return indices
+
+ def get_cm_ind_string_list(self):
+ cm_ind_string_list = []
+ if self.cloudmasks:
+ cm_ind_string_list.extend([cm["cm_string"] for cm in self.cloudmasks])
+ if self.indices:
+ cm_ind_string_list.extend([ind["indice_string"] for ind in self.indices])
+ return cm_ind_string_list
diff --git a/sen2chain/tiles.py b/sen2chain/tiles.py
index 47f8a0d..b0ba662 100644
--- a/sen2chain/tiles.py
+++ b/sen2chain/tiles.py
@@ -828,7 +828,7 @@ class Tile:
start: str = "2015-01-01",
end: str = "9999-12-31",
new: bool = False,
- ref: str = "l1c",
+ ref = "l1c",
min_cloudcover: int = 0,
max_cloudcover: int = 100,
order_offline: bool = False,
diff --git a/sen2chain/utils.py b/sen2chain/utils.py
index 008fc9f..1d8c22c 100644
--- a/sen2chain/utils.py
+++ b/sen2chain/utils.py
@@ -287,13 +287,13 @@ def get_indice_from_identifier(identifier) -> str:
indice = (identifier.replace(".", "_").split("_")[7]).upper()
return indice
-def get_cloudmask_indice_dict_from_strings(
- cm_string: str = None,
- indice_string: str = None,
-) -> dict:
- """Returns a cloudmas or indice dict for FamilyProduct class
- :param string: string from which to extract the version name.
- can be a cloudmask or an indice identifier
- """
- returned_val = None
+# def get_cloudmask_indice_dict_from_strings(
+ # cm_string: str = None,
+ # indice_string: str = None,
+# ) -> dict:
+ # """Returns a cloudmas or indice dict for FamilyProduct class
+ # :param string: string from which to extract the version name.
+ # can be a cloudmask or an indice identifier
+ # """
+ # returned_val = None
--
GitLab