diff --git a/sen2chain/download_eodag.py b/sen2chain/download_eodag.py
index 3283dffa2b5b974cbef9cf3e75a6bd91a2a5a442..01bdb4467f1a56d04159abee2d230e2e58400b70 100644
--- a/sen2chain/download_eodag.py
+++ b/sen2chain/download_eodag.py
@@ -21,6 +21,7 @@ import multiprocessing
from .config import SHARED_DATA, Config
from .utils import get_tile
+from .products import FamilyProduct
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@@ -99,6 +100,12 @@ class S2cEodag:
)
logging.disable(logging.NOTSET)
+ if not isinstance(ref, list):
+ ref_string = ref
+ else:
+ ref_string = "specific cloudmask(s) and / or indice(s) product(s)"
+ logger.info("Searching online products, ref: {}".format(ref))
+
fitered = self.products[:]
for p in fitered:
if (outputs_prefix / (p.properties["title"] + ".SAFE")).exists():
@@ -123,28 +130,121 @@ class S2cEodag:
p.properties["storageStatus"] = "ONLINE"
if not(min_cloudcover <= float(p.properties["cloudCover"] or 0) <= max_cloudcover):
self.products.remove(p)
- logger.info("{} - local l1c {} - filtering (CC = {}%)".format(p.properties["title"], l1c_presence, int(p.properties["cloudCover"])))
+ logger.info(
+ "{} - local l1c {} - filtering (CC = {}%)".format(
+ p.properties["title"],
+ l1c_presence,
+ int(p.properties["cloudCover"])
+ )
+ )
else:
if ref == "l1c":
- if (outputs_prefix / (p.properties["title"] + ".SAFE")).exists():
- logger.info("{} - local l1c {} - filtering".format(p.properties["title"], l1c_presence, p.properties["storageStatus"]))
+ if l1c_presence == "PRESENT":
+ logger.info(
+ "{} - l1c PRESENT - filtering".format(
+ p.properties["title"],
+ )
+ )
self.products.remove(p)
else:
- logger.info("{} - local l1c {} - remote {}".format(p.properties["title"], l1c_presence, p.properties["storageStatus"]))
+ logger.info(
+ "{} - l1c ABSENT - remote {}".format(
+ p.properties["title"],
+ p.properties["storageStatus"]
+ )
+ )
elif ref == "l2a":
- if (Path(Config().get("l2a_path")) / self.name / (p.properties["title"].replace("L1C_", "L2A_").replace("__OPER__", "_USER_") + ".SAFE")).exists():
- logger.info("{} - local l1c {} - local l2a {} - filtering".format(p.properties["title"], l1c_presence, l2a_presence, p.properties["storageStatus"]))
+ if l2a_presence == "PRESENT":
+ logger.info(
+ "{} - l2a PRESENT - filtering".format(
+ p.properties["title"],
+ )
+ )
self.products.remove(p)
else:
- if (outputs_prefix / (p.properties["title"] + ".SAFE")).exists():
- logger.info("{} - local l1c {} - filtering".format(p.properties["title"], l1c_presence, p.properties["storageStatus"]))
+ if l1c_presence == "PRESENT":
+ logger.info(
+ "{} - l1c PRESENT - filtering".format(
+ p.properties["title"],
+ )
+ )
self.products.remove(p)
else:
- logger.info("{} - local l1c {} - local l2a {} - remote {}".format(p.properties["title"], l1c_presence, l2a_presence, p.properties["storageStatus"]))
-
- # elif ref == "ndvi" or indice:
- # toto = 13
+ logger.info(
+ "{} - l1c ABSENT, l2a ABSENT - remote {}".format(
+ p.properties["title"],
+ p.properties["storageStatus"]
+ )
+ )
+ elif ref == "cloudmasks":
+ fp = FamilyProduct(p.properties["title"])
+ if fp.cloudmasks:
+ logger.info(
+ "{} - at least one cloudmask PRESENT - filtering".format(
+ p.properties["title"],
+ )
+ )
+ self.products.remove(p)
+ else:
+ if l2a_presence == "PRESENT":
+ logger.info(
+ "{} - l2a PRESENT - cloudmasks ABSENT - filtering".format(
+ p.properties["title"],
+ )
+ )
+ self.products.remove(p)
+ else:
+ if l1c_presence == "PRESENT":
+ logger.info(
+ "{} - l1c PRESENT - cloudmasks ABSENT - filtering".format(
+ p.properties["title"],
+ )
+ )
+ self.products.remove(p)
+ else:
+ logger.info(
+ "{} - l1c ABSENT, l2a ABSENT, cloudmasks ABSENT - remote {}".format(
+ p.properties["title"],
+ p.properties["storageStatus"]
+ )
+ )
+ elif ref == "indices":
+ fp = FamilyProduct(p.properties["title"])
+ if fp.indices:
+ logger.info(
+ "{} - at least one indice PRESENT - filtering".format(
+ p.properties["title"],
+ )
+ )
+ self.products.remove(p)
+ else:
+ if l2a_presence == "PRESENT":
+ logger.info(
+ "{} - l2a PRESENT - indices ABSENT - filtering".format(
+ p.properties["title"],
+ )
+ )
+ self.products.remove(p)
+ else:
+ if l1c_presence == "PRESENT":
+ logger.info(
+ "{} - l1c PRESENT - indices ABSENT - filtering".format(
+ p.properties["title"],
+ l1c_presence,
+ p.properties["storageStatus"]
+ )
+ )
+ self.products.remove(p)
+ else:
+ logger.info(
+ "{} - l1c ABSENT, l2a ABSENT, indices ABSENT - remote {}".format(
+ p.properties["title"],
+ p.properties["storageStatus"]
+ )
+ )
+ elif isinstance(ref, list):
+ toto =3
# clouds = self.products[:]
diff --git a/sen2chain/products.py b/sen2chain/products.py
index 0847440236f91231a0758eeda69cbb40ac086815..0331be24a22114a82baa184ce4469daa14d40c0b 100755
--- a/sen2chain/products.py
+++ b/sen2chain/products.py
@@ -1766,10 +1766,12 @@ class FamilyProduct(dict):
self.update(fid_tile)
self.family_id = fid_tile["family_id"]
self.tile = fid_tile["tile"]
- self.l1c_id = self.get_l1c()
- self.l2a_id = self.get_l2a()
- self.cloudmasks = self.get_cloudmasks()
- self.indices = self.get_indices()
+ self.l1c_id = self.get_l1c()
+ self.l2a_id = self.get_l2a()
+ self.cloudmasks = self.get_cloudmasks()
+ self.indices = self.get_indices()
+ else:
+ logger.info("Invalid identifier {}".format(identifier))
for key, val in self.__dict__.items():
self[key] = val
@@ -1805,36 +1807,42 @@ class FamilyProduct(dict):
def get_cloudmasks(self):
cloudmasks = []
- library_path = self._cloudmask_library_path / self.tile / self.l2a_id
- paths = list(library_path.glob("*L2A*" + self.family_id + "_CM*.jp2"))
- for item in paths:
- cm_dict = get_cm_dict(item.stem)
- cm_string = get_cm_string_from_dict(cm_dict)
- cm_dict["cm_string"] = cm_string
- cloudmasks.append(cm_dict)
+ try:
+ library_path = self._cloudmask_library_path / self.tile / self.l2a_id
+ paths = list(library_path.glob("*L2A*" + self.family_id + "_CM*.jp2"))
+ for item in paths:
+ cm_dict = get_cm_dict(item.stem)
+ cm_string = get_cm_string_from_dict(cm_dict)
+ cm_dict["cm_string"] = cm_string
+ cloudmasks.append(cm_dict)
+ except:
+ pass
return cloudmasks
def get_indices(self):
indices = []
- library_path = self._indice_library_path
- paths = list(
- library_path.glob(
- "*/"
- + self.tile + "/"
- + self.l2a_id + "/"
- + "*.jp2"
+ try:
+ library_path = self._indice_library_path
+ paths = list(
+ library_path.glob(
+ "*/"
+ + self.tile + "/"
+ + self.l2a_id + "/"
+ + "*.jp2"
+ )
)
- )
- for item in paths:
- logger.info(item)
- indice_dict = {}
- indice_dict["indice"] = get_indice_from_identifier(item.stem)
- cm_dict = get_cm_dict(item.stem)
- if cm_dict:
- indice_dict.update(cm_dict)
- indice_dict["cm_string"] = get_cm_string_from_dict(cm_dict)
- indice_dict["indice_string"] = indice_dict["indice"] + "_" + indice_dict["cm_string"]
- else:
- indice_dict["indice_string"] = indice_dict["indice"]
- indices.append(indice_dict)
+ for item in paths:
+ # logger.info(item)
+ indice_dict = {}
+ indice_dict["indice"] = get_indice_from_identifier(item.stem)
+ cm_dict = get_cm_dict(item.stem)
+ if cm_dict:
+ indice_dict.update(cm_dict)
+ indice_dict["cm_string"] = get_cm_string_from_dict(cm_dict)
+ indice_dict["indice_string"] = indice_dict["indice"] + "_" + indice_dict["cm_string"]
+ else:
+ indice_dict["indice_string"] = indice_dict["indice"]
+ indices.append(indice_dict)
+ except:
+ pass
return indices
diff --git a/sen2chain/tiles.py b/sen2chain/tiles.py
index f0666229affeba9e388186c5c1828a3b1dbbc5d5..47f8a0d58edc0ebb20a7f47b8d8dd21bfcfc58c6 100644
--- a/sen2chain/tiles.py
+++ b/sen2chain/tiles.py
@@ -835,6 +835,10 @@ class Tile:
tries: int = 1,
sleep: int = 0,
):
+ """
+ Function to download l1c products using EODAG
+ """
+
if provider == "scihub":
NUM_THREADS = 4
else: