some changes to Tiles

sen2chain/tiles.py

@@ -910,12 +910,15 @@ class Tile:
                     ) if before_list[i] != after_list[i]
                 ]
                 # logger.info("changed_list {}".format(changed_list))
-                if len(changed_list) == len(before_list):
-                    logger.info("Downloaded all {} product(s)".format(len(before_list)))
-                    break
+                if len(before_list):
+                    if len(changed_list) == len(before_list):
+                        logger.info("Downloaded all {} product(s)".format(len(before_list)))
+                        break
+                    else:
+                        logger.info("Downloaded {}/{} product(s)".format(len(changed_list), len(before_list)))
                 else:
-                    logger.info("Downloaded {}/{} product(s)".format(len(changed_list), len(before_list)))
-                    
+                    logger.info("No product to download")
+                        
                 if (tries > 1) and (essai < (tries-1)):
                     logger.info("Sleeping {}min before retrying".format(sleep))
                     time.sleep(sleep*60)
@@ -1195,7 +1198,7 @@ class Tile:
         remove_indice_tif: bool = False, 
         remove: bool = False,
         deep_search: bool = False,
-        tempo_depth: int = 0,   ##### to implement to limit deep search to last xx days
+        tempo_depth: int = 99999,   ##### to implement to limit deep search to last xx days
     ):
         """
         Function to search and clean corrupted files during processing errors
@@ -1205,17 +1208,17 @@ class Tile:
         - moved error l2a products from l2a folder
         - cloud masks error (0kb)
         - indices error (0kb)
-
+        
         :param clean_list: list of tiles to be cleaned. If not provided, will
             process all l1c Tiles.
         :param remove_indice_tif: bool. If True will remove present TIFF files
             present in indice folders. Default value False.
         :param remove: bool. If True will effectively remove corrupted files,
             if False will just list identified problems. Default value False.
-
+        
         """
         # logger.info("Cleaning {} library".format(self.name))
-
+        
         nb_id = 0
         nb_rm = 0
         prob_id = []
@@ -1262,29 +1265,33 @@ class Tile:
                         logger.error(
                             "Can't remove {} from L2A folder".format(f.name)
                         )
-        
-        # identify corrupted jp2 in l1c folder
         if deep_search:
+            # identify corrupted jp2 in l1c folder
             erase_set = set()
-            for f in self._paths["l1c"].glob("*/GRANULE/*/IMG_DATA/*.jp2"):
-                if f.stat().st_size == 0:
-                    txt = "Identified 0b corrupted {} in L1C folder".format(f.name)
-                    prob_id.append(txt)
-                    logger.info(txt)
-                    nb_id += 1
-                    if remove:
-                        erase_set.update({f.parent.parent.parent.parent})
-            for e in erase_set:
-                try:
-                    logger.info("Removing {} from L1C folder".format(e))
-                    shutil.rmtree(str(e))
-                    nb_rm += 1
-                except:
-                    logger.error("Can't remove {} from L1C folder".format(e))
+            for p in [p for p in t.l1c if p.date >= datetime.datetime.now() - datetime.timedelta(tempo_depth)]:
+                l1c = L1cProduct(p.identifier)
+                for f in l1c.path.glob("GRANULE/*/IMG_DATA/*.jp2"):
+                # for f in self._paths["l1c"].glob("*/GRANULE/*/IMG_DATA/*.jp2"):
+                    if f.stat().st_size == 0:
+                        txt = "Identified 0b corrupted {} in L1C folder".format(f.name)
+                        prob_id.append(txt)
+                        logger.info(txt)
+                        nb_id += 1
+                        if remove:
+                            erase_set.update({f.parent.parent.parent.parent})
+                for e in erase_set:
+                    try:
+                        logger.info("Removing {} from L1C folder".format(e))
+                        shutil.rmtree(str(e))
+                        nb_rm += 1
+                    except:
+                        logger.error("Can't remove {} from L1C folder".format(e))
             
-        # identify 0B cloud masks
-        if deep_search:
-            for f in self._paths["cloudmasks"].glob("*/*CM*.jp2"):
+            # identify 0B cloud masks
+            for p in [p for p in t.cloudmasks if p.date >= datetime.datetime.now() - datetime.timedelta(tempo_depth)]:
+                cm = NewCloudMaskProduct(p.identifier)
+                f = cm.path
+                # for f in self._paths["cloudmasks"].glob("*/*CM*.jp2"):
                 if f.stat().st_size == 0:
                     txt = "Corrupted cloud mask {} in L2A folder".format(f.name)
                     prob_id.append(txt)
@@ -1298,8 +1305,7 @@ class Tile:
                         )
                         f.unlink()
                         nb_rm += 1
-        # identify wrong size l2a_QL
-        if deep_search:
+            # identify wrong size l2a_QL
             for f in self._paths["l2a"].glob("QL/*_QL.tif"):
                 if f.stat().st_size != 3617212:
                     txt = "Corrupted L2A QL {} in L2A QL folder".format(f.name)
@@ -1314,10 +1320,21 @@ class Tile:
                         )
                         f.unlink()
                         nb_rm += 1
+                        
         # identify 0B or absent indice QL
+        """
         for f in self._paths["indices"]:
-            # logger.info(f, self._paths["indices"][f])
-            for p in self._paths["indices"][f].glob("*_MSIL2A_*/"):
+            indices = set(p.stem for p in list(self._paths["indices"][f].glob("*_MSIL2A_*/")))
+            
+            for fp in set(p.stem for p in list(self._paths["indices"][f].glob("*_MSIL2A_*/")) if FamilyProduct(p.stem).date >= datetime.datetime.now() - datetime.timedelta(tempo_depth)):
+                
+            for p in [p.family_id for p self._paths["indices"][f].glob("*_MSIL2A_*/") if ]
+            
+            
+            for indice in [p for p in getattr(self, f) if p.date >= datetime.datetime.now() - datetime.timedelta(tempo_depth)]:
+                ind = IndiceProduct(indi.identifier)
+                p = ind.path
+            # for p in self._paths["indices"][f].glob("*_MSIL2A_*/"):
                 # logger.info(p)
                 if p.is_file():
                     txt = "Identified old indice format {}".format(p.name)
@@ -1396,6 +1413,9 @@ class Tile:
                                 )
                                 q.unlink()
                                 nb_rm += 1
+        """
+        
+        
         return {
             "identified_problems": nb_id,
             "removed_problems": nb_rm,