From 9aadee4824174110376810dae8ef4c58d35d0fa1 Mon Sep 17 00:00:00 2001
From: Impact <pascal.mouquet@ird.fr>
Date: Mon, 28 Oct 2019 13:48:31 +0400
Subject: [PATCH] Remove raw TIFF from all indices folders / PIL is now used to
check images corruption in clean_lib
---
sen2chain/cloud_mask.py | 77 +----------------------------------------
sen2chain/indices.py | 49 ++++++++++++++++++++++----
sen2chain/tiles.py | 22 +++++++++---
3 files changed, 61 insertions(+), 87 deletions(-)
diff --git a/sen2chain/cloud_mask.py b/sen2chain/cloud_mask.py
index 5525ad6..eb70fbe 100755
--- a/sen2chain/cloud_mask.py
+++ b/sen2chain/cloud_mask.py
@@ -304,39 +304,6 @@ def create_cloud_mask_v2(
with rasterio.open(out_dilate, "w", **cld_profile) as dst:
dst.write(cld_dilate.astype(np.int8), 1)
- #~ # Seuillage du masque nuage Sen2Cor
- #~ CLD_seuil = 25
- #~ BandMath = otbApplication.Registry.CreateApplication("BandMath")
- #~ BandMath.SetParameterStringList("il", [str(cloud_mask)])
- #~ BandMath.SetParameterString("out", out_bin)
- #~ BandMath.SetParameterOutputImagePixelType("out", 0)
- #~ BandMath.SetParameterString("exp", "im1b1 > " + str(CLD_seuil))
- #~ BandMath.ExecuteAndWriteOutput()
-
- #~ # Erosion
- #~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
- #~ BinaryMorphologicalOperation.SetParameterString("in", out_bin)
- #~ BinaryMorphologicalOperation.SetParameterString("out", out_erode)
- #~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
- #~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", 1)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", 1)
- #~ BinaryMorphologicalOperation.SetParameterFloat("filter.erode.foreval", 1)
- #~ BinaryMorphologicalOperation.SetParameterString("filter", "erode")
- #~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
-
- #~ # Dilatation
- #~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
- #~ BinaryMorphologicalOperation.SetParameterString("in", out_erode)
- #~ BinaryMorphologicalOperation.SetParameterString("out", out_dilate)
- #~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
- #~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", dilatation)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", dilatation)
- #~ BinaryMorphologicalOperation.SetParameterFloat("filter.dilate.foreval", 1)
- #~ BinaryMorphologicalOperation.SetParameterString("filter", "dilate")
- #~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
-
# Save to JP2000
logger.info("converting to JP2000 file...")
src_ds = gdal.Open(out_dilate)
@@ -425,49 +392,7 @@ def create_cloud_mask_b11(
with rasterio.Env(GDAL_CACHEMAX=512) as env:
with rasterio.open(out_mask, "w", **cld_profile) as dst:
dst.write(cld_mskd_dilate.astype(np.int8), 1)
-
- #~ # Seuillage du masque B11
- #~ b11_seuil = 1500
- #~ BandMath = otbApplication.Registry.CreateApplication("BandMath")
- #~ BandMath.SetParameterStringList("il", [str(b11_path)])
- #~ BandMath.SetParameterString("out", out_bin)
- #~ BandMath.SetParameterOutputImagePixelType("out", 0)
- #~ BandMath.SetParameterString("exp", "im1b1 < " + str(b11_seuil))
- #~ BandMath.ExecuteAndWriteOutput()
-
- #~ # Dilatation
- #~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
- #~ BinaryMorphologicalOperation.SetParameterString("in", out_bin)
- #~ BinaryMorphologicalOperation.SetParameterString("out", out_dilate)
- #~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
- #~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", dilatation)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", dilatation)
- #~ BinaryMorphologicalOperation.SetParameterFloat("filter.dilate.foreval", 1)
- #~ BinaryMorphologicalOperation.SetParameterString("filter", "dilate")
- #~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
-
- # Masquage du masque
- #~ BandMath = otbApplication.Registry.CreateApplication("BandMath")
- #~ BandMath.SetParameterStringList("il", [str(cloud_mask),str(out_dilate)])
- #~ BandMath.SetParameterString("out", out_mask)
- #~ BandMath.SetParameterOutputImagePixelType("out", 0)
- #~ BandMath.SetParameterString("exp", "(im1b1 == 1) * (im2b1 == 0)")
- #~ BandMath.ExecuteAndWriteOutput()
-
- #~ # Dilatation
- #~ BinaryMorphologicalOperation = otbApplication.Registry.CreateApplication("BinaryMorphologicalOperation")
- #~ BinaryMorphologicalOperation.SetParameterString("in", out_mask)
- #~ BinaryMorphologicalOperation.SetParameterString("out", out_dilate)
- #~ BinaryMorphologicalOperation.SetParameterOutputImagePixelType("out", 0)
- #~ BinaryMorphologicalOperation.SetParameterInt("channel", 1)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.xradius", dilatation+1)
- #~ BinaryMorphologicalOperation.SetParameterInt("structype.ball.yradius", dilatation+1)
- #~ BinaryMorphologicalOperation.SetParameterFloat("filter.dilate.foreval", 1)
- #~ BinaryMorphologicalOperation.SetParameterString("filter", "dilate")
- #~ BinaryMorphologicalOperation.ExecuteAndWriteOutput()
-
-
+
# Save to JP2000
logger.info("converting to JP2000 file...")
src_ds = gdal.Open(out_mask)
diff --git a/sen2chain/indices.py b/sen2chain/indices.py
index 9d88a74..2c075f4 100644
--- a/sen2chain/indices.py
+++ b/sen2chain/indices.py
@@ -129,6 +129,7 @@ class Ndvi(Indice):
try:
os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
except:
pass
@@ -223,7 +224,13 @@ class NdwiGao(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
-
+
+ try:
+ os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
+ except:
+ pass
+
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
@@ -309,7 +316,13 @@ class NdwiMcf(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
-
+
+ try:
+ os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
+ except:
+ pass
+
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
@@ -399,7 +412,13 @@ class BIGR(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
-
+
+ try:
+ os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
+ except:
+ pass
+
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
@@ -482,7 +501,13 @@ class BIRNIR(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
-
+
+ try:
+ os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
+ except:
+ pass
+
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
@@ -565,7 +590,13 @@ class BIBG(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
-
+
+ try:
+ os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
+ except:
+ pass
+
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
@@ -646,7 +677,13 @@ class Mndwi(Indice):
index_tiff_2_jp2(img_path=(out_path / masked_indice_raw),
out_path=(out_path / masked_indice_filename))
os.remove(str(out_path / masked_indice_raw))
-
+
+ try:
+ os.remove(str(out_path / self.indice_raw))
+ logger.info("Removing {}".format(self.indice_raw))
+ except:
+ pass
+
if quicklook:
cmap = matplotlib_colormap_to_rgb(self.colormap, revers=False)
diff --git a/sen2chain/tiles.py b/sen2chain/tiles.py
index 9de76b9..3e6fb5c 100644
--- a/sen2chain/tiles.py
+++ b/sen2chain/tiles.py
@@ -8,6 +8,7 @@ import logging
import re
import fiona
import shutil
+from PIL import Image
from pathlib import Path
from collections import namedtuple
@@ -403,11 +404,22 @@ class Tile:
logger.info("Removing old indice format {}".format(p.name))
p.unlink()
else:
- for q in p.glob("*_QUICKLOOK.tif"):
- if not ((q.stat().st_size == 3617212) or
- (q.stat().st_size == 4196652) or
- (q.stat().st_size == 3617478)):
- logger.info("Corrupted indice QL {} (bad size)".format(q.name))
+ #~ for q in p.glob("*_QUICKLOOK.tif"):
+ #~ if not ((q.stat().st_size == 3617212) or
+ #~ (q.stat().st_size == 4196652) or
+ #~ (q.stat().st_size == 3617478)):
+ #~ logger.info("Corrupted indice QL {} (bad size)".format(q.name))
+ #~ if remove:
+ #~ logger.info("Removing indice QL {}".format(q.name))
+ #~ q.unlink()
+ for q in list(p.glob("*.jp2")) + list(p.glob("*.tif")):
+ #~ logger.info(q)
+ try:
+ Image.MAX_IMAGE_PIXELS = 120560400
+ img = Image.open(str(q)) # open the image file
+ img.verify() # verify that it is, in fact an image
+ except (IOError, SyntaxError) as e:
+ logger.info('Bad file (PIL): {}'.format(str(q.name))) # print out the names of corrupt files
if remove:
logger.info("Removing indice QL {}".format(q.name))
q.unlink()
--
GitLab