From 811d82231cb98ace5d337b47990fd4e225ac32aa Mon Sep 17 00:00:00 2001
From: Impact <pascal.mouquet@ird.fr>
Date: Mon, 12 Oct 2020 10:16:08 +0400
Subject: [PATCH] initiating sen2chain products versioning
---
sen2chain/__init__.py | 6 +-
sen2chain/config.py | 16 +-
sen2chain/data/sen2chain_info.xml | 7 +
sen2chain/products.py | 297 +++++++++++++++++++++++++++++-
sen2chain/sen2cor.py | 8 +-
sen2chain/tiles.py | 184 +++++++++++++++---
sen2chain/time_series.py | 2 +-
sen2chain/utils.py | 8 +-
sen2chain/xmlparser.py | 64 ++++++-
9 files changed, 544 insertions(+), 48 deletions(-)
create mode 100644 sen2chain/data/sen2chain_info.xml
diff --git a/sen2chain/__init__.py b/sen2chain/__init__.py
index 9d15b50..c2b715c 100644
--- a/sen2chain/__init__.py
+++ b/sen2chain/__init__.py
@@ -21,16 +21,16 @@ This module lists all externally useful classes and functions.
from .config import Config
from .tiles import Tile
-from .products import L1cProduct, L2aProduct
+from .products import L1cProduct, L2aProduct, OldCloudMaskProduct, NewCloudMaskProduct
from .library import Library
from .data_request import DataRequest
from .indices import IndicesCollection
from .download_and_process import DownloadAndProcess
from .time_series import TimeSeries
from .automatization import Automatization
-from .utils import format_word, grouper, datetime_to_str, str_to_datetime, human_size_decimal, human_size
+from .utils import format_word, grouper, datetime_to_str, str_to_datetime, human_size_decimal, human_size, get_current_Sen2Cor_version
from .geo_utils import serialise_tiles_index, get_processed_indices_vect, crop_product_by_shp
from .multi_processing import l2a_multiprocessing, cldidx_multiprocessing, cld_multiprocessing, idx_multiprocessing
from .tileset import TileSet
-__version__ = "0.1.0"
+__version__ = "0.6.0"
__author__ = "Jérémy Commins <jebins@openmailbox.org> & Impact <pascal.mouquet@ird.fr>"
diff --git a/sen2chain/config.py b/sen2chain/config.py
index 5047fb5..0404a5c 100644
--- a/sen2chain/config.py
+++ b/sen2chain/config.py
@@ -16,9 +16,10 @@ logging.basicConfig(level=logging.INFO)
ROOT = Path(os.path.realpath(__file__)).parent.parent
SHARED_DATA = dict(
- tiles_index=ROOT / "sen2chain" / "data" / "tiles_index.gpkg",
- tiles_index_dict=ROOT / "sen2chain" / "data" / "tiles_index_dict.p",
- peps_download=ROOT / "sen2chain" / "peps_download3.py")
+ tiles_index = ROOT / "sen2chain" / "data" / "tiles_index.gpkg",
+ tiles_index_dict = ROOT / "sen2chain" / "data" / "tiles_index_dict.p",
+ peps_download = ROOT / "sen2chain" / "peps_download3.py",
+ sen2chain_meta = ROOT / "sen2chain" / "data" / "sen2chain_info.xml")
class Config:
@@ -42,18 +43,21 @@ class Config:
self._config_params = ConfigParser()
self._config_params["DATA PATHS"] = {"temp_path": "",
"l1c_path": "",
- "l1c_archive_path": "/ARCHIVE_SENTINEL-PROD/S2_L1C_ARCHIVE",
+ "l1c_archive_path": "",
"l2a_path": "",
- "l2a_archive_path": "/ARCHIVE_SENTINEL-PROD/S2_L2A_ARCHIVE",
+ "l2a_archive_path": "",
"indices_path": "",
"time_series_path": "",
- "temporal_summaries_path": ""}
+ "temporal_summaries_path": "",
+ "cloudmasks_path": "",
+ }
self._config_params["SEN2COR PATH"] = {"sen2cor_bashrc_path": ""}
self._config_params["HUBS LOGINS"] = {"scihub_id": "",
"scihub_pwd": "",
"peps_config_path": ""}
self._config_params["PROXY SETTINGS"] = {"proxy_http_url": "",
"proxy_https_url": ""}
+ self._config_params["SEN2CHAIN VERSIONS"] = {"sen2chain_processing_version": "xx.xx"}
if self._CONFIG_FILE.exists():
self._config_params.read(str(self._CONFIG_FILE))
diff --git a/sen2chain/data/sen2chain_info.xml b/sen2chain/data/sen2chain_info.xml
new file mode 100644
index 0000000..05bcdbe
--- /dev/null
+++ b/sen2chain/data/sen2chain_info.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Sen2Chain_Report_File>
+ <SEN2CHAIN_VERSION></SEN2CHAIN_VERSION>
+ <SEN2CHAIN_PROCESSING_VERSION></SEN2CHAIN_PROCESSING_VERSION>
+ <SEN2COR_VERSION></SEN2COR_VERSION>
+</Sen2Chain_Report_File>
+
diff --git a/sen2chain/products.py b/sen2chain/products.py
index e59cc13..3a35bc2 100755
--- a/sen2chain/products.py
+++ b/sen2chain/products.py
@@ -13,14 +13,15 @@ from pathlib import Path
# type annotations
from typing import List, Tuple, Optional
-from .utils import grouper, setPermissions
+from .utils import grouper, setPermissions, get_current_Sen2Cor_version
from .config import Config, SHARED_DATA
-from .xmlparser import MetadataParser
+from .xmlparser import MetadataParser, Sen2ChainMetadataParser
from .sen2cor import process_sen2cor
from .cloud_mask import create_cloud_mask, create_cloud_mask_v2, create_cloud_mask_b11
from .indices import IndicesCollection
from .colormap import create_l2a_ql, create_l1c_ql
+
s2_tiles_index = SHARED_DATA.get("tiles_index")
logging.basicConfig(level=logging.INFO)
@@ -133,7 +134,10 @@ class Product:
:param key: tag of the metadata to search.
"""
self._update_metadata_parser()
- return self._metadata_parser.get_band_path(key=key, res=res)
+ try:
+ return self._metadata_parser.get_band_path(key=key, res=res)
+ except:
+ return None
@property
def in_library(self) -> bool:
@@ -207,6 +211,7 @@ class L1cProduct(Product):
if self._library_product:
l2a_prod.archive()
L2aProduct(l2a_identifier).setPermissions()
+ L2aProduct(l2a_identifier).update_sen2chain_metadata()
return self
def process_ql(self,
@@ -363,35 +368,44 @@ class L2aProduct(Product):
_library_path = Path(Config().get("l2a_path"))
_indices_library_path = Path(Config().get("indices_path"))
_tiled_metadata = "MTD_MSIL2A.xml"
-
+
def __init__(
self,
identifier: str = None,
tile: str = None,
path: str = None
) -> None:
-
+ logger.info("l2a-001")
super().__init__(identifier=identifier, tile=tile, path=path)
if not re.match(r".*L2A_.*", identifier):
raise ValueError("Invalid L2A product name")
self.indices_path = self._indices_library_path
-
+ logger.info("l2a-002")
# user cloud mask
self.user_cloud_mask = self.path.parent / (self.identifier + "_CLOUD_MASK.jp2")
if not self.user_cloud_mask.exists():
self.user_cloud_mask = None
-
+ logger.info("l2a-003")
# user cloud mask b11
self.user_cloud_mask_b11 = self.path.parent / (self.identifier + "_CLOUD_MASK_B11.jp2")
if not self.user_cloud_mask_b11.exists():
self.user_cloud_mask_b11 = None
-
+ logger.info("l2a-004")
# user QL
self.user_ql = self.path.parent / (self.identifier + "_QL.tif")
if not self.user_ql.exists():
self.user_ql = None
-
+
+ logger.info("l2a-005")
+ # versions
+ self._sen2chain_info_path = self.path / "sen2chain_info.xml"
+ logger.info("l2a-005-00")
+ if self._sen2chain_info_path.parent.exists() and not self._sen2chain_info_path.exists():
+ logger.info("l2a-005-01")
+ Sen2ChainMetadataParser(self._sen2chain_info_path).init_metadata()
+ logger.info("l2a-006")
+
def process_ql(self,
reprocess: bool = False,
out_path: pathlib.PosixPath = None,
@@ -502,6 +516,62 @@ class L2aProduct(Product):
self.user_cloud_mask_b11 = cloud_mask_b11_path
return self
+
+
+ def compute_cloud_mask(self,
+ version: str = "cm001",
+ probability: int = 1,
+ iterations: int = 5,
+ #~ buffering: bool = True,
+ reprocess: bool = False,
+ out_path_mask = None,
+ out_path_mask_b11 = None
+ ) -> "L2aProduct":
+ """
+ """
+ logger.info("{}: processing cloud_mask_v3".format(self.identifier))
+
+ cloudmask = NewCloudMaskProduct(l2a_identifier = self.identifier,
+ sen2chain_processing_version = self.sen2chain_processing_version,
+ version = version,
+ probability = probability,
+ iterations = iterations)
+
+ if cloudmask.path and not reprocess:
+ logger.info("{} cloud mask already exists".format(cloudmask.identifier))
+ else:
+ if cloudmask.updated_path:
+ if version == "cm001":
+ if cloudmask.path:
+ cloudmask.path.unlink() # in version 3.8 will be updated using missing_ok = True
+ create_cloud_mask_v2(self.cld_20m,
+ erosion=1,
+ dilatation=5,
+ out_path=cloudmask.updated_path)
+ self.user_cloud_mask = cloudmask.updated_path
+ elif version == "cm002":
+ cloudmask_cm001 = NewCloudMaskProduct(l2a_identifier = self.identifier,
+ sen2chain_processing_version = self.sen2chain_processing_version,
+ version = "cm001")
+ if cloudmask_cm001.path:
+ if cloudmask.path:
+ cloudmask.path.unlink() # in version 3.8 will be updated using missing_ok = True
+ create_cloud_mask_b11(cloudmask_cm001.path,
+ self.b11_20m,
+ dilatation=5,
+ out_path=Path(str(cloudmask_cm001.path).replace("CM001", "CM002-B11")))
+ else:
+ logger.info("No cloudmask version cm001 found, please compute this one first")
+ elif version == "cm003":
+ toto=12
+ elif version == "cm004":
+ toto=12
+ else:
+ logger.info("Wrong cloudmask version {}".format(version))
+ else:
+ logger.info("Impossible to compute cloudmask (no L2a product)")
+
+ return self
def process_indices(self,
indices_list: List[str] = [],
@@ -565,6 +635,48 @@ class L2aProduct(Product):
# indice_filename = template.format(product_identifier=self.identifier,
# ext=ext)
# return (self.indices_path / indice / self.tile / indice_filename).exists()
+
+ #~ @property
+ #~ def sen2cor_version(self):
+ #~ """ Used Sen2Cor version"""
+ #~ return Sen2ChainMetadataParser(self.identifier):
+
+
+ #~ sen2chain_info_path = self.path / "Sen2Chain_info.xml"
+ #~ if sen2chain_info_path.exists():
+ #~ return "xml present"
+ #~ Sen2ChainMetadataParser(self.identifier)
+
+ #~ else:
+ #~ return None
+
+ def update_md(self,
+ sen2chain_version: str = None,
+ sen2chain_processing_version: str = None,
+ sen2cor_version: str = None,
+ ):
+ """ Set custom sen2chain, sen2chain_processing and sen2cor versions """
+ Sen2ChainMetadataParser(self._sen2chain_info_path).set_metadata(sen2chain_version = sen2chain_version,
+ sen2chain_processing_version = sen2chain_processing_version,
+ sen2cor_version = sen2cor_version)
+
+
+ @property
+ def sen2chain_version(self):
+ return Sen2ChainMetadataParser(self._sen2chain_info_path).get_metadata_value('SEN2CHAIN_VERSION')
+
+ @property
+ def sen2chain_processing_version(self):
+ logger.info("SPV-001")
+ return Sen2ChainMetadataParser(self._sen2chain_info_path).get_metadata_value('SEN2CHAIN_PROCESSING_VERSION')
+
+ @property
+ def sen2cor_version(self):
+ return Sen2ChainMetadataParser(self._sen2chain_info_path).get_metadata_value('SEN2COR_VERSION')
+
+ @property
+ def generation_time(self):
+ return self._get_metadata_value(key="GENERATION_TIME")
# METADATA
@property
@@ -822,3 +934,170 @@ class L2aProduct(Product):
@property
def tci_60m(self):
return self._get_band_path(key="TCI", res="60m")
+
+
+class OldCloudMaskProduct:
+ """Old cloud mask product class.
+
+ :param identifier: cloudmask filename.
+
+ """
+ _library_path = Path(Config().get("l2a_path"))
+
+ def __init__(self, identifier: str = None) -> None:
+ if identifier is None:
+ raise ValueError("Product identifier is empty")
+ else:
+ self.identifier = identifier
+ self.tile = self.get_tile(identifier)
+ self.l2a = self.get_l2a(identifier)
+ self.path = self._library_path / self.tile / self.identifier
+
+
+ @staticmethod
+ def get_tile(identifier) -> str:
+ """Returns tile name from a string.
+
+ :param string: string from which to extract the tile name.
+ """
+ return re.findall("_T([0-9]{2}[A-Z]{3})_", identifier)[0]
+
+ @staticmethod
+ def get_l2a(identifier) -> str:
+ """Returns l2a name from a old cloud mask identifier string.
+ :param string: string from which to extract the l2a name.
+ """
+ return re.findall(r"(S2.+)_CLOUD_MASK.+jp2", identifier)[0]
+
+
+class NewCloudMaskProduct:
+ """New cloud mask product class.
+
+ :param identifier: cloudmask filename.
+
+ """
+ _library_path = Path(Config().get("cloudmasks_path"))
+
+ def __init__(self,
+ identifier: str = None,
+ l2a_identifier: str = None,
+ sen2chain_processing_version: str = None,
+ version: str = "cm001",
+ probability: int = 1,
+ iterations: int = 5,
+ ) -> None:
+ if not (identifier or l2a_identifier):
+ raise ValueError("Product or L2a identifier cannot be empty")
+ else:
+ self.tile = self.get_tile(identifier or l2a_identifier)
+ self.l2a = l2a_identifier.replace(".SAFE", "") or self.get_l2a(identifier)
+ self.path = None
+ self.updated_path = None
+ self.spv = None
+ self.uptodate = False
+ self.updatable = False
+ #~ self.theorical_path = None
+ if identifier:
+ self.identifier = identifier
+ else:
+ self.identifier = None
+ if version == "cm001":
+ suffix = "_CM001"
+ elif version == "cm002":
+ suffix = "_CM002-B11"
+ elif version == "cm003":
+ suffix = "_CM003-PRB" + str(probability) + "ITR" + str(iterations)
+ else:
+ raise ValueError("Cloud Mask version cmxxx is not defined")
+ logger.info("002")
+
+ self.path = next(iter(list((self._library_path / self.tile / self.l2a).glob(self.l2a.replace(".SAFE", "") + \
+ "_SPV*" + suffix + ".jp2"))), \
+ None)
+ if self.path:
+ self.identifier = self.path.name
+ self.spv = re.findall("_SPV([0-9]{4})_", str(self.path))[0]
+ if self.spv == Config().get("sen2chain_processing_version"):
+ self.uptodate = True
+ else:
+ l2a = L2aProduct(self.l2a)
+ if l2a.path.exists() and int(l2a.sen2chain_processing_version) > int(self.spv):
+ self.updatable = True
+
+ l2a = L2aProduct(self.l2a)
+ if l2a.path.exists():
+ self.updated_path = self._library_path /\
+ self.tile /\
+ self.l2a /\
+ (self.l2a + "_SPV" + l2a.sen2chain_processing_version +\
+ suffix + ".jp2")
+
+
+ #~ self.uptodate = # vrai ou faux si les deux sont égaux et si spv sup
+ #~ self.updatable = # vrai si uptidater et l2a exist
+
+
+
+
+ #~ l2a_product = L2aProduct(l2a_identifier)
+ #~ if not sen2chain_processing_version:
+ #~ logger.info("002-01")
+ #~ path = list((self._library_path / l2a_product.tile / self.l2a).glob(self.l2a.replace(".SAFE", "") + \
+ #~ "_SPV*" + \
+ #~ suffix + \
+ #~ ".jp2"))
+ #~ logger.info(path)
+ #~ if not path:
+ #~ try:
+ #~ sen2chain_processing_version = l2a_product.sen2chain_processing_version
+ #~ logger.info("sen2chain_processing_version {}".format(sen2chain_processing_version))
+ #~ self.identifier = self.l2a.replace(".SAFE", "") + \
+ #~ "_SPV" + str(sen2chain_processing_version or Config().get("sen2chain_processing_version")) + \
+ #~ suffix + \
+ #~ ".jp2"
+ #~ self.l2a_present = False
+
+ #~ except:
+ #~ self.l2a_present = True
+ #~ self.spv=None
+ #~ self.identifier = self.l2a.replace(".SAFE", "") + \
+ #~ "_SPV" + Config().get("sen2chain_processing_version") + \
+ #~ suffix + \
+ #~ ".jp2"
+
+ #~ else:
+ #~ self.identifier = path[0].name
+ #~ self.spv = re.findall("_SPV([0-9]{4})_", self.identifier)[0]
+ #~ logger.info("003")
+
+
+
+ #~ else:
+ #~ self.identifier = identifier
+ #~ self.spv = re.findall("_SPV([0-9]{4})_", identifier)[0]
+ #~ self.l2a = self.get_l2a(self.identifier)
+ #~ logger.info("004")
+
+ #~ logger.info("005")
+
+ #~ logger.info("006")
+ #~ self.path = self._library_path / self.tile / self.l2a / self.identifier
+ #~ logger.info(self.path)
+ #~ if self.path.exists() and
+
+
+ @staticmethod
+ def get_tile(identifier) -> str:
+ """Returns tile name from a string.
+
+ :param string: string from which to extract the tile name.
+ """
+ return re.findall("_T([0-9]{2}[A-Z]{3})_", identifier)[0]
+
+ @staticmethod
+ def get_l2a(identifier) -> str:
+ """Returns l2a name from a old cloud mask identifier string.
+ :param string: string from which to extract the l2a name.
+ """
+ return re.findall(r"(S2.+)_SPV.+jp2", identifier)[0]
+
diff --git a/sen2chain/sen2cor.py b/sen2chain/sen2cor.py
index 85ac0f1..c88cce6 100644
--- a/sen2chain/sen2cor.py
+++ b/sen2chain/sen2cor.py
@@ -7,10 +7,11 @@ Module for sen2cor processings.
import logging
import pathlib
import subprocess
-import re
+#~ import re
from typing import Union
-from .config import Config
+#~ from .config import Config
+from .utils import get_current_Sen2Cor_version
logger = logging.getLogger(__name__)
@@ -36,7 +37,8 @@ def process_sen2cor(
# TODO: Add 60m resolution.
sen2cor_bashrc_path = Config().get("sen2cor_bashrc_path")
- s2c_v = next(iter(re.findall('Sen2Cor-(\d{2}\.\d{2}\.\d{2})', str(sen2cor_bashrc_path))), None)
+ #~ s2c_v = next(iter(re.findall('Sen2Cor-(\d{2}\.\d{2}\.\d{2})', str(sen2cor_bashrc_path))), None)
+ s2c_v = get_current_Sen2Cor_version()
l2a_product_path_tmp = l2a_product_path.parent / (l2a_product_path.stem + '.tmp')
if s2c_v == '02.05.05':
diff --git a/sen2chain/tiles.py b/sen2chain/tiles.py
index 2f5155a..d0ae050 100644
--- a/sen2chain/tiles.py
+++ b/sen2chain/tiles.py
@@ -22,7 +22,7 @@ from itertools import chain
from .config import Config, SHARED_DATA
from .utils import str_to_datetime, human_size, getFolderSize
from .indices import IndicesCollection
-from .products import L1cProduct, L2aProduct
+from .products import L1cProduct, L2aProduct, OldCloudMaskProduct, NewCloudMaskProduct
from .multi_processing import l2a_multiprocessing, cld_multiprocessing, idx_multiprocessing
logging.basicConfig(level=logging.INFO)
@@ -155,6 +155,47 @@ class IndicesList(ProductsList):
return filtered
+class CloudMaskList(ProductsList):
+ """Class for managing mask product list
+
+ """
+ @property
+ def cm001(self) -> "CloudMaskList":
+ filtered = CloudMaskList()
+ for k, v in self._dict.items():
+ if "_CM001" in k:
+ filtered[k] = {"date": v["date"], "cloud_cover": v["cloud_cover"]}
+ return filtered
+
+ @property
+ def cm002(self) -> "CloudMaskList":
+ filtered = CloudMaskList()
+ for k, v in self._dict.items():
+ if "_CM002" in k:
+ filtered[k] = {"date": v["date"], "cloud_cover": v["cloud_cover"]}
+ return filtered
+
+ @property
+ def cm003(self) -> "CloudMaskList":
+ filtered = CloudMaskList()
+ for k, v in self._dict.items():
+ if "_CM003" in k:
+ filtered[k] = {"date": v["date"], "cloud_cover": v["cloud_cover"]}
+ return filtered
+
+ def params(self,
+ probability: int = 1,
+ iterations: int = 5,
+ ):
+ filtered = CloudMaskList()
+ for k, v in self._dict.items():
+ if "_CM003" in k:
+ if "-PRB" + str(probability) + "ITER" + str(iterations) in k:
+ filtered[k] = {"date": v["date"], "cloud_cover": v["cloud_cover"]}
+ else:
+ filtered[k] = {"date": v["date"], "cloud_cover": v["cloud_cover"]}
+ return filtered
+
class Tile:
"""Class for managing tiles in the library.
@@ -167,19 +208,23 @@ class Tile:
self.name = name
self._paths = {"l1c": Path(Config().get("l1c_path")) / name,
"l2a": Path(Config().get("l2a_path")) / name,
- "indices": {}
+ "indices": {},
+ "cloudmasks": Path(Config().get("cloudmasks_path")) / name,
}
self._indices_path = Path(Config().get("indices_path"))
-
+ self._cloudmasks_path = Path(Config().get("cloudmasks_path"))
+
self._products = {"l1c": ProductsList(),
"l2a": ProductsList(),
"cloudmasks" : ProductsList(),
+ "cloudmasks2" : ProductsList(),
"indices": dict()}
self._get_indices_paths()
self._get_l1c_list()
self._get_l2a_list()
self._get_cloudmasks()
+ self._get_cloudmasks2()
self._get_indices_list()
def _get_indices_paths(self) -> None:
@@ -217,6 +262,22 @@ class Tile:
cloud_cover = L2aProduct(f.name.replace("_CLOUD_MASK.jp2", ".SAFE"), self.name).cloud_coverage_assessment
self._products["cloudmasks"][f.name.replace("_CLOUD_MASK.jp2", ".SAFE")] = {"date": date, "cloud_cover": float(cloud_cover)}
+ def _get_cloudmasks2(self) -> None:
+ """Scans cloudmasks folder for cloud masks and adds corresponding L2A products in a ProductsList."""
+ self._products["cloudmasks2"] = CloudMaskList()
+ for f in self._paths["cloudmasks"].glob("*L2A*/*_CM*.jp2"):
+ l1c_name = f.parent.name.replace("L2A_", "L1C_").replace("_USER_", "_OPER_") + ".SAFE"
+ try:
+ date = self._products["l1c"][l1c_name].date
+ cloud_cover = self._products["l1c"][l1c_name].cloud_cover
+ except KeyError:
+ date = Tile._get_date(f.parent.name)
+ cloud_cover = L2aProduct(f.parent.name, self.name).cloud_coverage_assessment
+ self._products["cloudmasks2"][f.name] = {"date": date,
+ "cloud_cover": float(cloud_cover),
+ #~ "version": re.findall(r"_(CM...)", f.name)[0]
+ }
+
def _get_indices_list(self) -> None:
"""Scans indices folders and adds products in a IndicesList."""
for indice, path in self._paths["indices"].items():
@@ -269,7 +330,7 @@ class Tile:
def __str__(self) -> str:
return self.name
-
+
@property
def paths(self) -> Dict[str, pathlib.PosixPath]:
"""Returns all the paths related to a Tile object."""
@@ -290,6 +351,11 @@ class Tile:
"""Returns tile's cloud masks products as a ProductsList."""
return self._products["cloudmasks"]
+ @property
+ def cloudmasks2(self) -> "ProductsList":
+ """Returns tile's cloud masks products as a ProductsList."""
+ return self._products["cloudmasks2"]
+
@property
def l1c_missings(self) -> "ProductsList":
"""Returns tile's L2A products that don't have a L1C as a ProductsList."""
@@ -321,6 +387,22 @@ class Tile:
prods_list[prod] = {"date": self._products["l2a"][prod].date,
"cloud_cover": self._products["l2a"][prod].cloud_cover}
return prods_list
+
+ def cloudmasks_missing(self,
+ version: str = "cm001",
+ probability: int = 1,
+ iterations: int = 5,
+ ) -> "ProductsList":
+ """Returns tile's L2A products that don't have a cloud mask as a ProductsList."""
+ prods_list = ProductsList()
+ missings_l2a_set = set(self.l2a.products) - {(re.findall(r"(S2.+)_SPV.+.jp2", identifier)[0] + ".SAFE") \
+ for identifier in getattr(self.cloudmasks2, version).\
+ params(probability = probability, iterations = iterations).\
+ products}
+ for prod in missings_l2a_set:
+ prods_list[prod] = {"date": self._products["l2a"][prod].date,
+ "cloud_cover": self._products["l2a"][prod].cloud_cover}
+ return prods_list
@property
def info(self):
@@ -332,22 +414,24 @@ class Tile:
@property
def size(self):
- #~ logger.info("Product: local / archived / total".format(human_size(getFolderSize(str(self.paths['l1c'])))))
- local = getFolderSize(str(self.paths["l1c"]))
- total = getFolderSize(str(self.paths["l1c"]), True)
- logger.info("l1c: {} (local: {} / archived: {})".format(human_size(total),
- human_size(local),
- human_size(total-local),
- ))
-
- #~ logger.info("l2a: {}".format(human_size(getFolderSize(str(self.paths["l2a"]), True))))
- local = getFolderSize(str(self.paths["l2a"]))
- total = getFolderSize(str(self.paths["l2a"]), True)
- logger.info("l2a: {} (local: {} / archived: {})".format(human_size(total),
- human_size(local),
- human_size(total-local),
- ))
-
+ try:
+ local = getFolderSize(str(self.paths["l1c"]))
+ total = getFolderSize(str(self.paths["l1c"]), True)
+ logger.info("l1c: {} (local: {} / archived: {})".format(human_size(total),
+ human_size(local),
+ human_size(total-local),
+ ))
+ except:
+ pass
+ try:
+ local = getFolderSize(str(self.paths["l2a"]))
+ total = getFolderSize(str(self.paths["l2a"]), True)
+ logger.info("l2a: {} (local: {} / archived: {})".format(human_size(total),
+ human_size(local),
+ human_size(total-local),
+ ))
+ except:
+ pass
for indice, path in self._paths["indices"].items():
logger.info("{}: {}".format(indice, human_size(getFolderSize(str(path), True))))
@@ -384,9 +468,12 @@ class Tile:
l2a_res = l2a_multiprocessing(l1c_process_list, nb_proc=nb_proc)
def compute_cloudmasks(self,
- date_min: str = None,
- date_max: str = None,
- nb_proc: int = 4):
+ version: str = "cm001",
+ probability: int = 1,
+ iterations: int = 5,
+ date_min: str = None,
+ date_max: str = None,
+ nb_proc: int = 4):
"""
Compute all missing cloud masks for l2a products
"""
@@ -402,7 +489,31 @@ class Tile:
cld_res = False
if cld_l2a_process_list:
cld_res = cld_multiprocessing(cld_l2a_process_list, nb_proc=nb_proc)
-
+
+ #~ def compute_cloudmasks2(self,
+ #~ version: str = "cm001",
+ #~ probability: int = 1,
+ #~ iterations: int = 5,
+ #~ date_min: str = None,
+ #~ date_max: str = None,
+ #~ nb_proc: int = 4):
+ #~ """
+ #~ Compute all missing cloud masks for l2a products
+ #~ """
+
+ #~ cld_l2a_process_list = []
+ #~ cld_l2a_process_list.append(list(p.identifier for p in self.cloudmasks_missings.filter_dates(date_min = date_min, date_max = date_max)))
+ #~ cld_l2a_process_list = list(chain.from_iterable(cld_l2a_process_list))
+ #~ if cld_l2a_process_list:
+ #~ logger.info("{} l2a products to process:".format(len(cld_l2a_process_list)))
+ #~ logger.info("{}".format(cld_l2a_process_list))
+ #~ else:
+ #~ logger.info("All cloud masks already computed")
+ #~ cld_res = False
+ #~ if cld_l2a_process_list:
+ #~ cld_res = cld_multiprocessing(cld_l2a_process_list, nb_proc=nb_proc)
+
+
def compute_indices(self,
indices: list = [],
date_min: str = None,
@@ -674,3 +785,28 @@ class Tile:
else:
l2a.process_ql(out_path = outfullpath, out_resolution=(750,750), jpg = True)
+ def update_cloudmasks(self):
+ """
+ Move and rename old cloudmasks to new cloudmask folder
+ cloudmask xmls are removed
+ """
+ #Move and rename old masks / B11
+ logger.info("Moving and renaming old masks")
+ for f in self._paths["l2a"].glob("*L2A*_CLOUD_MASK*.jp2"):
+ p = OldCloudMaskProduct(f.name)
+ f_renamed = f.name.replace("CLOUD_MASK_B11",
+ "SPV" + \
+ L2aProduct(p.l2a).sen2chain_processing_version.replace(".", "") +\
+ "_CM002-B11")\
+ .replace("CLOUD_MASK",
+ "SPV" + \
+ L2aProduct(p.l2a).sen2chain_processing_version.replace(".", "") +\
+ "_CM001")\
+ p_new = NewCloudMaskProduct(f_renamed)
+ p_new.path.parent.mkdir(exist_ok=True)
+ p.path.replace(p_new.path)
+
+ #Remove xml
+ logger.info("Removing xmls")
+ for f in self._paths["l2a"].glob("*L2A*_CLOUD_MASK*.jp2.aux.xml"):
+ f.unlink()
diff --git a/sen2chain/time_series.py b/sen2chain/time_series.py
index a7911c4..c29bc67 100644
--- a/sen2chain/time_series.py
+++ b/sen2chain/time_series.py
@@ -41,7 +41,7 @@ from .products import L2aProduct
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
#~ logging.basicConfig(format='%(process)s %(asctime)s %(levelname)s:%(module)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
-logging.getLogger().handlers[0].setFormatter(logging.Formatter('%(process)s %(asctime)s %(levelname)s:%(name)s:%(message)s',
+logging.getLogger().handlers[0].setFormatter(logging.Formatter('%(process)s:%(asctime)s:%(levelname)s:%(name)s:%(message)s',
'%Y-%m-%d %H:%M:%S'))
class TimeSeries:
diff --git a/sen2chain/utils.py b/sen2chain/utils.py
index bdd95fb..4b235a2 100644
--- a/sen2chain/utils.py
+++ b/sen2chain/utils.py
@@ -9,7 +9,9 @@ import logging
from itertools import zip_longest
from datetime import datetime
from typing import Iterable
+import re
+from .config import Config
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
@@ -110,7 +112,11 @@ def setPermissions(path):
os.chmod(name, os.stat(name).st_mode | 0o064)
os.chmod(name, os.stat(name).st_mode & ~0o003)
+def get_current_Sen2Cor_version():
+ """ Returns your current Sen2Cor version """
+ sen2cor_bashrc_path = Config().get("sen2cor_bashrc_path")
+ return next(iter(re.findall('Sen2Cor-(\d{2}\.\d{2}\.\d{2})', str(sen2cor_bashrc_path))), None)
+
-
diff --git a/sen2chain/xmlparser.py b/sen2chain/xmlparser.py
index 6f0e918..b3f20d6 100644
--- a/sen2chain/xmlparser.py
+++ b/sen2chain/xmlparser.py
@@ -9,11 +9,15 @@ import pathlib
import re
import xml.etree.ElementTree as et
+import sen2chain
+from .utils import get_current_Sen2Cor_version
+from .config import SHARED_DATA, Config
+
+
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
-
class MetadataParser:
"""Class for getting metadata values from a L1C or L2A product's XML metadata file.
@@ -126,3 +130,61 @@ class MetadataParser:
except IndexError:
logger.error("Band not found: {}".format(pattern))
raise
+
+class Sen2ChainMetadataParser:
+ """
+
+ """
+ def __init__(self,
+ xml_path,
+ ):
+ SEN2CHAIN_META = SHARED_DATA.get("sen2chain_meta")
+ logger.info("Sen2ChainMetadataParser-001")
+ self.xml_path = xml_path
+
+ if xml_path.exists():
+ logger.info("Sen2ChainMetadataParser-002")
+ self._tree = et.parse(str(xml_path))
+ logger.info("Sen2ChainMetadataParser-003")
+
+ else:
+ logger.info("Sen2ChainMetadataParser-004")
+ self._tree = et.parse(str(SEN2CHAIN_META))
+ logger.info("Sen2ChainMetadataParser-005")
+
+ self._root = self._tree.getroot()
+
+ def get_default_values(self):
+ keys = ["SEN2CHAIN_VERSION", "SEN2CHAIN_PROCESSING_VERSION", "SEN2COR_VERSION"]
+ self.default_values = []
+ for key in keys:
+ self.default_values.append([v.text for v in self._root.findall(".//{0}".format(key))][0])
+
+ def get_metadata_value(self, key: str = None) -> str:
+ """
+ Returns metadata.
+
+ :param key: metadata tag name.
+ """
+ try:
+ logger.info("get_metadata_value-001")
+ return [v.text for v in self._root.findall(".//{0}".format(key))][0]
+ except IndexError:
+ logger.error("Metadata value not found: {}".format(key))
+
+ def init_metadata(self):
+ self._root.find("SEN2CHAIN_VERSION").text = sen2chain.__version__
+ self._root.find("SEN2CHAIN_PROCESSING_VERSION").text = Config().get("sen2chain_processing_version")
+ self._root.find("SEN2COR_VERSION").text = get_current_Sen2Cor_version()
+ self._tree.write(str(self.xml_path), encoding="UTF-8", xml_declaration=True)
+
+ def set_metadata(self,
+ sen2chain_version: str = None,
+ sen2chain_processing_version: str = None,
+ sen2cor_version: str = None,
+ ):
+ self._root.find("SEN2CHAIN_VERSION").text = sen2chain_version or sen2chain.__version__
+ self._root.find("SEN2CHAIN_PROCESSING_VERSION").text = sen2chain_processing_version or Config().get("sen2chain_processing_version")
+ self._root.find("SEN2COR_VERSION").text = sen2cor_version or get_current_Sen2Cor_version()
+ self._tree.write(str(self.xml_path), encoding="UTF-8", xml_declaration=True)
+
--
GitLab