Skip to content
Snippets Groups Projects
Commit 9aadee48 authored by pascal.mouquet_ird.fr's avatar pascal.mouquet_ird.fr
Browse files

Remove raw TIFF from all indices folders / PIL is now used to check images corruption in clean_lib

parent c4812451
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
sen2chain/cloud_mask.py
+ 1
− 76
View file @ 9aadee48
......@@ -304,39 +304,6 @@ def create_cloud_mask_v2(
with rasterio.open(out_dilate, "w", **cld_profile) as dst:
dst.write(cld_dilate.astype(np.int8), 1)
#~ # Seuillage du masque nuage Sen2Cor
#~ CLD_seuil = 25
#~ BandMath = otbApplication.Registry.CreateApplication("BandMath")
#~ BandMath.SetParameterStringList("il", [str(cloud_mask)])
#~ BandMath.SetParameterString("out", out_bin)
#~ BandMath.SetParameterOutputImagePixelType("out", 0)
#~ BandMath.SetParameterString("exp", "im1b1 > " + str(CLD_seuil))
#~ BandMath.ExecuteAndWriteOutput()
#~ # Erosion
#~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
#~ BinaryMorphologicalOperation.SetParameterString("in", out_bin)
#~ BinaryMorphologicalOperation.SetParameterString("out", out_erode)
#~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
#~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", 1)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", 1)
#~ BinaryMorphologicalOperation.SetParameterFloat("filter.erode.foreval", 1)
#~ BinaryMorphologicalOperation.SetParameterString("filter", "erode")
#~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
#~ # Dilatation
#~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
#~ BinaryMorphologicalOperation.SetParameterString("in", out_erode)
#~ BinaryMorphologicalOperation.SetParameterString("out", out_dilate)
#~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
#~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", dilatation)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", dilatation)
#~ BinaryMorphologicalOperation.SetParameterFloat("filter.dilate.foreval", 1)
#~ BinaryMorphologicalOperation.SetParameterString("filter", "dilate")
#~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
# Save to JP2000
logger.info("converting to JP2000 file...")
src_ds = gdal.Open(out_dilate)
......@@ -425,49 +392,7 @@ def create_cloud_mask_b11(
with rasterio.Env(GDAL_CACHEMAX=512) as env:
with rasterio.open(out_mask, "w", **cld_profile) as dst:
dst.write(cld_mskd_dilate.astype(np.int8), 1)
#~ # Seuillage du masque B11
#~ b11_seuil = 1500
#~ BandMath = otbApplication.Registry.CreateApplication("BandMath")
#~ BandMath.SetParameterStringList("il", [str(b11_path)])
#~ BandMath.SetParameterString("out", out_bin)
#~ BandMath.SetParameterOutputImagePixelType("out", 0)
#~ BandMath.SetParameterString("exp", "im1b1 < " + str(b11_seuil))
#~ BandMath.ExecuteAndWriteOutput()
#~ # Dilatation
#~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
#~ BinaryMorphologicalOperation.SetParameterString("in", out_bin)
#~ BinaryMorphologicalOperation.SetParameterString("out", out_dilate)
#~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
#~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", dilatation)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", dilatation)
#~ BinaryMorphologicalOperation.SetParameterFloat("filter.dilate.foreval", 1)
#~ BinaryMorphologicalOperation.SetParameterString("filter", "dilate")
#~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
# Masquage du masque
#~ BandMath = otbApplication.Registry.CreateApplication("BandMath")
#~ BandMath.SetParameterStringList("il", [str(cloud_mask),str(out_dilate)])
#~ BandMath.SetParameterString("out", out_mask)
#~ BandMath.SetParameterOutputImagePixelType("out", 0)
#~ BandMath.SetParameterString("exp", "(im1b1 == 1) * (im2b1 == 0)")
#~ BandMath.ExecuteAndWriteOutput()
#~ # Dilatation
#~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
#~ BinaryMorphologicalOperation.SetParameterString("in", out_mask)
#~ BinaryMorphologicalOperation.SetParameterString("out", out_dilate)
#~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
#~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", dilatation+1)
#~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", dilatation+1)
#~ BinaryMorphologicalOperation.SetParameterFloat("filter.dilate.foreval", 1)
#~ BinaryMorphologicalOperation.SetParameterString("filter", "dilate")
#~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
# Save to JP2000
logger.info("converting to JP2000 file...")
src_ds = gdal.Open(out_mask)
......
sen2chain/indices.py
+ 43
− 6
View file @ 9aadee48
......@@ -129,6 +129,7 @@ class Ndvi(Indice):
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
......@@ -223,7 +224,13 @@ class NdwiGao(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
......@@ -309,7 +316,13 @@ class NdwiMcf(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
......@@ -399,7 +412,13 @@ class BIGR(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
......@@ -482,7 +501,13 @@ class BIRNIR(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
......@@ -565,7 +590,13 @@ class BIBG(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
......@@ -646,7 +677,13 @@ class Mndwi(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
try:
os.remove(str(out_path / self.indice_raw))
logger.info("Removing {}".format(self.indice_raw))
except:
pass
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
......
sen2chain/tiles.py
+ 17
− 5
View file @ 9aadee48
......@@ -8,6 +8,7 @@ import logging
import re
import fiona
import shutil
from PIL import Image
from pathlib import Path
from collections import namedtuple
......@@ -403,11 +404,22 @@ class Tile:
logger.info("Removing old indice format {}".format(p.name))
p.unlink()
else:
for q in p.glob("*_QUICKLOOK.tif"):
if not ((q.stat().st_size == 3617212) or
(q.stat().st_size == 4196652) or
(q.stat().st_size == 3617478)):
logger.info("Corrupted indice QL {} (bad size)".format(q.name))
#~ for q in p.glob("*_QUICKLOOK.tif"):
#~ if not ((q.stat().st_size == 3617212) or
#~ (q.stat().st_size == 4196652) or
#~ (q.stat().st_size == 3617478)):
#~ logger.info("Corrupted indice QL {} (bad size)".format(q.name))
#~ if remove:
#~ logger.info("Removing indice QL {}".format(q.name))
#~ q.unlink()
for q in list(p.glob("*.jp2")) + list(p.glob("*.tif")):
#~ logger.info(q)
try:
Image.MAX_IMAGE_PIXELS = 120560400
img = Image.open(str(q)) # open the image file
img.verify() # verify that it is, in fact an image
except (IOError, SyntaxError) as e:
logger.info('Bad file (PIL): {}'.format(str(q.name))) # print out the names of corrupt files
if remove:
logger.info("Removing indice QL {}".format(q.name))
q.unlink()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment