job fix

sen2chain/download_eodag.py

@@ -116,7 +116,8 @@ class S2cEodag:
                 self.products.remove(p)
                 logger.info("{} - local l1c {} - filtering (wrong Tile)".format(p.properties["title"], l1c_presence))
             else:
-                if not(min_cloudcover <= float(p.properties["cloudCover"]) <= max_cloudcover):
+                logger.info(p.properties["cloudCover"])
+                if not(min_cloudcover <= float(p.properties["cloudCover"] or 0) <= max_cloudcover):
                     self.products.remove(p)
                     logger.info("{} - local l1c {} - filtering (CC = {}%)".format(p.properties["title"], l1c_presence, int(p.properties["cloudCover"])))
                 else:

sen2chain/jobs.py

@@ -187,6 +187,7 @@ class Job:
                         )
                     else:
                         logger.info("{} not found".format(arg))
+                self.clean_fields()
                 logger.info("\n{}".format(self.tasks))
             else:
                 logger.info("Task_number not found")
@@ -236,6 +237,7 @@ class Job:
             for line in header:
                 ict.write(line)
             # self.tasks.to_csv(ict)
+            self.unclean_fields()
             self.tasks.to_csv(ict, index=False, sep=";")
 
     def get_cron_status(self):
@@ -344,7 +346,9 @@ class Job:
         ##### prévoir ici un check pour les champs nouveaux ou manquants !
         if 'comments' not in self.tasks:
             self.tasks['comments'] = ""
-        
+        self.clean_fields()
+    
+    def clean_fields(self):
         for index, row in self.tasks.iterrows():
             if not row.date_min:
                 # self.tasks.at[index, "start_time"] = (datetime.datetime.now()-datetime.timedelta(days=delta_t)).strftime('%Y-%m-%d')
@@ -363,14 +367,16 @@ class Job:
                         index, "indices"
                     ] = IndicesCollection.available_indices
                 else:
-                    self.tasks.at[index, "indices"] = str(row.indices).split(
-                        "/"
-                    )
-            if not row.cloudmask == "False":
-                self.tasks.at[index, "cloudmask"] = self.get_cm_version(
-                    row.cloudmask
-                )
+                    if not isinstance(self.tasks.at[index, "indices"] , list):
+                        self.tasks.at[index, "indices"] = str(row.indices).split(
+                            "/"
+                        )
 
+    def unclean_fields(self):
+        for index, row in self.tasks.iterrows():
+            if (not row.indices == "False") and (not row.indices == "All"):
+                    self.tasks.at[index, "indices"] = '/'.join(self.tasks.at[index, "indices"])   
+    
     @staticmethod
     def get_cm_version(identifier) -> str:
         """Returns cloudmask version from a cloud mask identifier string.
@@ -378,7 +384,7 @@ class Job:
         """
         try:
             pat = re.compile(r"(?P<cm_version>CM00[1-2])")
-            return pat.match(identifier).groupdict()
+            # return pat.match(identifier).groupdict()
         except:
             pass
         try:
@@ -387,7 +393,7 @@ class Job:
                 + "-PRB(?P<probability>.*)"
                 + "-ITER(?P<iterations>.*)"
             )
-            return pat.match(identifier).groupdict()
+            # return pat.match(identifier).groupdict()
         except:
             pass
         try:
@@ -399,9 +405,37 @@ class Job:
                 + "-TCI(?P<thin_cir>.*)"
                 + "-ITER(?P<iterations>.*)"
             )
-            return pat.match(identifier).groupdict()
+            # return pat.match(identifier).groupdict()
         except:
             pass
+        try:
+            returned_val = pat.match(identifier).groupdict()
+            if "probability" not in returned_val:
+                returned_val["probability"] = 1
+            if "iterations" not in returned_val:
+                returned_val["iterations"] = 5
+            if "cld_shad" not in returned_val:
+                returned_val["cld_shad"] = True
+            if "cld_med_prob" not in returned_val:
+                returned_val["cld_med_prob"] = True
+            if "cld_hi_prob" not in returned_val:
+                returned_val["cld_hi_prob"] = True
+            if "thin_cir" not in returned_val:
+                returned_val["thin_cir"] = True
+        except:
+            returned_val = {
+                "cm_version": "CM001",
+                "probability": 1,
+                "iterations": 5,
+                "cld_shad": True,
+                "cld_med_prob": True,
+                "cld_hi_prob": True, 
+                "thin_cir": True, 
+            }
+        
+        
+        return returned_val
+
 
     def run(
         self,
@@ -532,6 +566,7 @@ class Job:
             for index, row in self.tasks.iterrows():
                 # if not bool(distutils.util.strtobool(str(row.cloudmask))):
                 if not (row.cloudmask == "False" or not row.cloudmask):
+                    cloudmask = self.get_cm_version(row.cloudmask)                    
                     t = Tile(row.tile)
                     l2a_to_process = [
                         p.identifier
@@ -542,13 +577,13 @@ class Job:
                     for j in l2a_to_process:
                         l2a_cm_details = [
                             j,
-                            row.cloudmask["cm_version"],
-                            row.cloudmask["probability"],
-                            row.cloudmask["iterations"],
-                            row.cloudmask["cld_shad"],
-                            row.cloudmask["cld_med_prob"],
-                            row.cloudmask["cld_hi_prob"],
-                            row.cloudmask["thin_cir"],
+                            cloudmask["cm_version"],
+                            cloudmask["probability"],
+                            cloudmask["iterations"],
+                            cloudmask["cld_shad"],
+                            cloudmask["cld_med_prob"],
+                            cloudmask["cld_hi_prob"],
+                            cloudmask["thin_cir"],
                             reprocess,
                         ]
                         cld_l2a_process_list.append(l2a_cm_details)
@@ -602,13 +637,13 @@ class Job:
                                 reprocess,
                                 nodata_clouds,
                                 quicklook,
-                                row.cloudmask["cm_version"],
-                                row.cloudmask["probability"],
-                                row.cloudmask["iterations"],
-                                row.cloudmask["cld_shad"],
-                                row.cloudmask["cld_med_prob"],
-                                row.cloudmask["cld_hi_prob"],
-                                row.cloudmask["thin_cir"],
+                                cloudmask["cm_version"],
+                                cloudmask["probability"],
+                                cloudmask["iterations"],
+                                cloudmask["cld_shad"],
+                                cloudmask["cld_med_prob"],
+                                cloudmask["cld_hi_prob"],
+                                cloudmask["thin_cir"],
                             ]
                             indices_l2a_process_list.append(l2a_ind_details)
                         logger.info(