Skip to content
Snippets Groups Projects
Commit a3fbd57a authored by TGermain's avatar TGermain
Browse files

Merge branch 'master' of https://framagit.org/espace-dev/sen2chain

parents fe766560 82129dfd
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 89 additions and 75 deletions
sen2chain/tiles.py
+ 89
− 75
View file @ a3fbd57a
......@@ -1191,7 +1191,12 @@ class Tile:
else:
logger.info("All indices already computed")
def clean_lib(self, remove_indice_tif: bool = False, remove: bool = False):
def clean_lib(self,
remove_indice_tif: bool = False,
remove: bool = False,
deep_search: bool = False,
tempo_depth: int = 0, ##### to implement to limit deep search to last xx days
):
"""
Function to search and clean corrupted files during processing errors
from Tile :
......@@ -1211,26 +1216,10 @@ class Tile:
"""
# logger.info("Cleaning {} library".format(self.name))
# identify corrupted jp2 in l1c folder
nb_id = 0
nb_rm = 0
prob_id = []
erase_set = set()
for f in self._paths["l1c"].glob("*/GRANULE/*/IMG_DATA/*.jp2"):
if f.stat().st_size == 0:
txt = "Identified 0b corrupted {} in L1C folder".format(f.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
erase_set.update({f.parent.parent.parent.parent})
for e in erase_set:
try:
logger.info("Removing {} from L1C folder".format(e))
shutil.rmtree(str(e))
nb_rm += 1
except:
logger.error("Can't remove {} from L1C folder".format(e))
# identify residual l2a from l1c folder
for f in chain(
self._paths["l1c"].glob("*L2A*.SAFE"),
......@@ -1249,6 +1238,7 @@ class Tile:
logger.error(
"Can't remove {} from L1C folder".format(f.name)
)
# identify missing jp2 in L2A folder
for f in self._paths["l2a"].glob("*L2A*.SAFE"):
# Nb jp2 < 7
......@@ -1272,36 +1262,58 @@ class Tile:
logger.error(
"Can't remove {} from L2A folder".format(f.name)
)
# identify corrupted jp2 in l1c folder
if deep_search:
erase_set = set()
for f in self._paths["l1c"].glob("*/GRANULE/*/IMG_DATA/*.jp2"):
if f.stat().st_size == 0:
txt = "Identified 0b corrupted {} in L1C folder".format(f.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
erase_set.update({f.parent.parent.parent.parent})
for e in erase_set:
try:
logger.info("Removing {} from L1C folder".format(e))
shutil.rmtree(str(e))
nb_rm += 1
except:
logger.error("Can't remove {} from L1C folder".format(e))
# identify 0B cloud masks
for f in self._paths["cloudmasks"].glob("*/*CM*.jp2"):
if f.stat().st_size == 0:
txt = "Corrupted cloud mask {} in L2A folder".format(f.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing corrupted cloud mask {} from L2A folder".format(
f.name
if deep_search:
for f in self._paths["cloudmasks"].glob("*/*CM*.jp2"):
if f.stat().st_size == 0:
txt = "Corrupted cloud mask {} in L2A folder".format(f.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing corrupted cloud mask {} from L2A folder".format(
f.name
)
)
)
f.unlink()
nb_rm += 1
f.unlink()
nb_rm += 1
# identify wrong size l2a_QL
for f in self._paths["l2a"].glob("QL/*_QL.tif"):
if f.stat().st_size != 3617212:
txt = "Corrupted L2A QL {} in L2A QL folder".format(f.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing corrupted QL {} from L2A folder".format(
f.name
if deep_search:
for f in self._paths["l2a"].glob("QL/*_QL.tif"):
if f.stat().st_size != 3617212:
txt = "Corrupted L2A QL {} in L2A QL folder".format(f.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing corrupted QL {} from L2A folder".format(
f.name
)
)
)
f.unlink()
nb_rm += 1
f.unlink()
nb_rm += 1
# identify 0B or absent indice QL
for f in self._paths["indices"]:
# logger.info(f, self._paths["indices"][f])
......@@ -1327,25 +1339,26 @@ class Tile:
# if remove:
# logger.info("Removing indice QL {}".format(q.name))
# q.unlink()
for q in list(p.glob("*.jp2")) + list(p.glob("*.tif")):
# logger.info(q)
try:
Image.MAX_IMAGE_PIXELS = 120560400
img = Image.open(str(q)) # open the image file
img.verify() # verify that it is, in fact an image
except (IOError, SyntaxError) as e:
txt = "Bad file (PIL): {}".format(
str(q.name)
) # print out the names of corrupt files
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing indice QL {}".format(q.name)
)
q.unlink()
nb_rm += 1
if deep_search:
for q in list(p.glob("*.jp2")) + list(p.glob("*.tif")):
# logger.info(q)
try:
Image.MAX_IMAGE_PIXELS = 120560400
img = Image.open(str(q)) # open the image file
img.verify() # verify that it is, in fact an image
except (IOError, SyntaxError) as e:
txt = "Bad file (PIL): {}".format(
str(q.name)
) # print out the names of corrupt files
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing indice QL {}".format(q.name)
)
q.unlink()
nb_rm += 1
for q in list(p.glob("*.jp2")):
if not (Path(str(q) + ".aux.xml")).exists():
txt = "Missing metadata: {}".format(q.name)
......@@ -1356,18 +1369,19 @@ class Tile:
logger.info("Removing jp2 {}".format(q.name))
q.unlink()
nb_rm += 1
for q in p.glob("*.*"):
if q.stat().st_size == 0:
txt = "Corrupted file {} (0B size)".format(q.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing indice QL {}".format(q.name)
)
q.unlink()
nb_rm += 1
if deep_search:
for q in p.glob("*.*"):
if q.stat().st_size == 0:
txt = "Corrupted file {} (0B size)".format(q.name)
prob_id.append(txt)
logger.info(txt)
nb_id += 1
if remove:
logger.info(
"Removing indice QL {}".format(q.name)
)
q.unlink()
nb_rm += 1
if remove_indice_tif:
for q in p.glob("*" + f.upper() + ".tif"):
txt = "Identified indice in tif format {}".format(
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment