Merge branch 'master' of https://framagit.org/espace-dev/sen2chain

sen2chain/jobs.py

@@ -725,6 +725,12 @@ class Job:
                         f.write("Sleeping {}min before retrying\n".format(self.sleep))
                         f.flush()
                     time.sleep(self.sleep*60)
+            
+            # remove duplicates from all_downloaded_products list
+            all_downloaded_products.sort()
+            uniq = list(k for k,_ in groupby(all_downloaded_products))
+            all_downloaded_products = uniq
+            
             logger.info(
                 "Total {}/{} downloaded product(s)".format(
                     len(all_downloaded_products),
@@ -773,7 +779,7 @@ class Job:
                 # )
             # )
             
-            ### remove duplicate l2a
+            ### remove duplicate l1c from l1c_process_list
             l1c_process_list.sort()
             uniq = list(k for k,_ in groupby(l1c_process_list))
             l1c_process_list = uniq
@@ -791,6 +797,7 @@ class Job:
                 )
                 f.flush()
             
+            ### Computing L2A
             l2a_res = False
             if l1c_process_list:
                 l2a_res = l2a_multiprocessing(
@@ -802,20 +809,43 @@ class Job:
                 if "l1c" in str(row.remove).lower():
                     t = Tile(row.tile)
                     prodlist = [p for p in all_downloaded_products if row.tile in p]
+                    prodlist_cleaned = [
+                        p for p in prodlist if p in [Path(q.identifier).stem for q in t.l1c]
+                    ]
+                    prodlist_skipped = [
+                        p for p in prodlist if p not in [Path(q.identifier).stem for q in t.l1c]
+                    ]
                     logger.info(
                         "{} removing downloaded l1c product(s): {}".format(
                             row.tile,
-                            prodlist,
+                            prodlist_cleaned,
                         )
                     )
+                    if prodlist_skipped:
+                        logger.info(
+                            "{} skipping {} l1c product(s), download error?:\n{}".format(
+                                row.tile,
+                                len(prodlist_skipped),
+                                prodlist_skipped,
+                            )
+                        )                    
                     if self.logs:
                         f.write(
-                            "{}\nRemoving downloaded l1c products for tile: {}\n{}\n\n".format(
+                            "{}\nRemoving downloaded l1c products for tile: {}\n{}\n".format(
                                 datetime.datetime.now(), 
                                 row.tile,
-                                prodlist,
+                                prodlist_cleaned,
                             )
                         )
+                        if prodlist_skipped:
+                            f.write(
+                                "\nSkipping {} l1c product(s), download error?:\n{}".format(
+                                    len(prodlist_skipped),
+                                    prodlist_skipped,
+                                )
+                            )  
+                        else:
+                            f.write("\n")
                         f.flush()                      
                     t.remove_l1c(prodlist)
 
@@ -1014,23 +1044,46 @@ class Job:
                 if "l2a" in str(row.remove).lower():
                     t = Tile(row.tile)
                     prodlist = [p for p in l2a_remove_list if row.tile in p]
+                    prodlist_cleaned = [
+                        p for p in prodlist if p in [Path(q.identifier).stem for q in t.l2a]
+                    ]
+                    prodlist_skipped = [
+                        p for p in prodlist if p not in [Path(q.identifier).stem for q in t.l2a]
+                    ]
                     logger.info(
                         "{} removing produced l2a product(s): {}".format(
                             row.tile,
-                            prodlist,
+                            prodlist_cleaned,
                         )
                     )
+                    if prodlist_skipped:
+                        logger.info(
+                            "{} skipping {} l2a product(s), Sen2Cor error?:\n{}".format(
+                                row.tile,
+                                len(prodlist_skipped),
+                                prodlist_skipped,
+                            )
+                        )                          
                     if self.logs:
                         f.write(
-                            "{}\nRemoving produced l2a products for tile: {}\n{}\n\n".format(
+                            "{}\nRemoving produced l2a products for tile: {}\n{}\n".format(
                                 datetime.datetime.now(), 
                                 row.tile,
-                                prodlist,
+                                prodlist_cleaned,
                             )
                         )
+                        if prodlist_skipped:
+                            f.write(
+                                "\nSkipping {} l2a product(s), download error?:\n{}".format(
+                                    len(prodlist_skipped),
+                                    prodlist_skipped,
+                                )
+                            )  
+                        else:
+                            f.write("\n")
                         f.flush()                      
                     t.remove_l2a(prodlist)
-                        
+
             # Cleaning after
             if clean_after or self.clean_after:
                 logger.info("Cleaning Tiles")