diff --git a/sen2chain/data/tiles_index.gpkg b/sen2chain/data/tiles_index.gpkg
index ca1f02ab10897923f24f6e9b6369a1bd1f75d9f1..96ff5c1332d802fc360e7cef528eee4620d07ea0 100644
Binary files a/sen2chain/data/tiles_index.gpkg and b/sen2chain/data/tiles_index.gpkg differ
diff --git a/sen2chain/download_eodag.py b/sen2chain/download_eodag.py
index 74b55254916f7cc5e974f18434f5a944702832c0..ad4352248233a6e35f3ab3dac788a844c5cbbf5e 100644
--- a/sen2chain/download_eodag.py
+++ b/sen2chain/download_eodag.py
@@ -18,6 +18,13 @@ from queue import Queue
from eodag.utils.logging import get_logging_verbose
from threading import Thread
import multiprocessing
+from simplejson import JSONDecodeError
+from urllib3.exceptions import (
+ ConnectTimeoutError, MaxRetryError,
+ NewConnectionError, SSLError,
+ ReadTimeoutError,
+)
+from requests.exceptions import ConnectTimeout
from .config import SHARED_DATA, Config
from .utils import get_tile, set_permissions
@@ -40,6 +47,7 @@ class S2cEodag:
self.provider = provider
self.products = None
logging.disable(level=logging.WARNING)
+ # os.environ["EODAG__PEPS__SEARCH__TIMEOUT"] = 50 # decomment when eodag > 2.11
self.dag = EODataAccessGateway()
logging.disable(logging.NOTSET)
@@ -68,21 +76,32 @@ class S2cEodag:
end = end,
tileIdentifier = self.name,
)
-
+ logger.info("{} - Searching online products, ref: {}".format(self.name, ref))
logging.disable(level=logging.WARNING)
- self.products = self.dag.search_all(
- **default_search_criteria
- )
- logging.disable(logging.NOTSET)
+ try:
+ self.products = self.dag.search_all(
+ **default_search_criteria
+ )
+ logging.disable(logging.NOTSET)
+ except (ConnectTimeout, MaxRetryError, TimeoutError, ConnectTimeoutError):
+ logging.disable(logging.NOTSET)
+ logger.info("Issue with this tile search for now, should try to increase the timeout for this provider or check your internet connexion")
+ self.products = []
+ pass
+ except:
+ logging.disable(logging.NOTSET)
+ logger.info("An error occured check everything")
+ self.products = []
+ pass
+
# if not isinstance(ref, list):
# ref_string = ref
# else:
# ref_string = "specific cloudmask(s) and / or indice(s) product(s)"
- logger.info("Searching online products, ref: {}".format(ref))
- fitered = self.products[:]
- for p in fitered:
+ filtered = self.products[:]
+ for p in filtered:
if (outputs_prefix / (p.properties["title"] + ".SAFE")).exists():
l1c_presence = "PRESENT"
else:
@@ -285,6 +304,7 @@ class S2cEodag:
outputs_prefix: str = None,
extract: bool = True,
delete_archive: bool = True,
+ remove_existing_zipfile: bool = True,
):
if not outputs_prefix:
if "L1C" in product_id.properties['title']:
@@ -294,24 +314,31 @@ class S2cEodag:
outputs_prefix = str(Path(Config().get(root_path)) / self.name)
setup_logging(verbose = 2)
-
+
+ if remove_existing_zipfile:
+ zip_file = L1cProduct(product_id.properties["id"]).zip_path
+ if zip_file:
+ logger.info("Found existing zipfile {}".format(zip_file.name))
+ try:
+ L1cProduct(product_id.properties["id"]).remove(zipfile = True)
+ except:
+ logger.info("Issue with removing {}".format(zip_file))
+ pass
if product_id.properties["storageStatus"] == "ONLINE":
# logging.disable(level=logging.WARNING)
- try:
- L1cProduct(product_id.properties["id"]).remove(zipfile = True)
- downloaded_path = self.dag.download(
- product_id,
- outputs_prefix = outputs_prefix,
- extract = extract,
- delete_archive = delete_archive,
- wait=1,
- timeout=0,
- )
- except:
- downloaded_path = None
- logger.info("Issue with {}, removing corrupted zipfile".format(product_id.properties["id"]))
- L1cProduct(product_id.properties["id"]).remove(zipfile = True)
- pass
+ # try:
+ downloaded_path = self.dag.download(
+ product_id,
+ outputs_prefix = outputs_prefix,
+ extract = extract,
+ delete_archive = delete_archive,
+ wait=1,
+ timeout=0,
+ )
+ # except:
+ # logger.info("Issue with dowloading {}, removing corrupted zipfile".format(product_id.properties["id"]))
+ # downloaded_path = None
+ # pass
if downloaded_path:
logger.info("dl_path {}".format(downloaded_path))
# logging.disable(logging.NOTSET)
@@ -331,7 +358,7 @@ class S2cEodag:
# shutil.rmtree(downloaded_path ,ignore_errors = True)
# logger.info("Moving up files and deleting old folder")
# downloaded_path=Path(downloaded_path).parent
- set_permissions(Path(downloaded_path))
+ set_permissions(Path(downloaded_path))
elif product_id.properties["storageStatus"] == "OFFLINE":
try:
diff --git a/sen2chain/products.py b/sen2chain/products.py
index 46cf7de71e1fd4e142d6bed3e73c82e65445e602..6c36adb067f706948db358d7a2691e41c9228da8 100755
--- a/sen2chain/products.py
+++ b/sen2chain/products.py
@@ -427,7 +427,11 @@ class L1cProduct(Product):
@property
def zip_path(self) -> str:
"""Returns zip path"""
- return self.path.parent / (self.path.stem + ".zip")
+ zip_path = self.path.parent / (self.path.stem + ".zip")
+ if zip_path.exists():
+ return zip_path
+ else:
+ return None
# METADATA
@property
diff --git a/sen2chain/tiles.py b/sen2chain/tiles.py
index e84aeeb682244658aa785e8576e897908fd56a0a..66addabf46d521f2283989e2db149d3e08ad42ee 100644
--- a/sen2chain/tiles.py
+++ b/sen2chain/tiles.py
@@ -1421,6 +1421,51 @@ class Tile:
"Can't remove {} from L1C folder".format(f.name)
)
+ # research residual zipfiles in l1c folder
+ for f in chain(
+ self._paths["l1c"].glob("*L1C*.zip"),
+ # self._paths["l1c"].glob("*L2A*.tmp"),
+ ):
+ txt = "Identified remaining {} in L1C folder".format(f.name)
+ prob_id.append(txt)
+ logger.info(txt)
+ nb_id += 1
+ if remove:
+ # try:
+ f.unlink()
+ logger.info("Removing {} from L1C folder".format(f.name))
+ nb_rm += 1
+ # except:
+ # logger.error(
+ # "Can't remove {} from L1C folder".format(f.name)
+ # )
+
+
+ # identify corrupted zip extractions from l1c folder
+ for f in chain(
+ self._paths["l1c"].glob("*L1C*"),
+ # self._paths["l1c"].glob("*L2A*.tmp"),
+ ):
+ if not f.suffix:
+ txt = "Identified {} corrupted zip extraction in L1C folder".format(f.name)
+ prob_id.append(txt)
+ logger.info(txt)
+ nb_id += 1
+ if remove:
+ if f.is_dir():
+ try:
+ shutil.rmtree(str(f))
+ logger.info("Removing {} folder from L1C folder".format(f.name))
+ nb_rm += 1
+ except:
+ logger.error(
+ "Can't remove {} from L1C folder".format(f.name)
+ )
+ else:
+ f.unlink()
+ logger.info("Removing {} file from L1C folder".format(f.name))
+ nb_rm += 1
+
# identify missing jp2 in L2A folder
for f in self._paths["l2a"].glob("*L2A*.SAFE"):
# Nb jp2 < 7