Skip to content
Snippets Groups Projects
Commit 3c82aba9 authored by pascal.mouquet_ird.fr's avatar pascal.mouquet_ird.fr
Browse files

generation of cloudmasks and their application in jobs seems to be OK now

parent 09315b3e
No related branches found
No related tags found
No related merge requests found
Pipeline #1331 passed
Stage: test
    Stage: deploy
      Hide whitespace changes
      Inline Side-by-side
      Showing
      with 75 additions and 58 deletions
      sen2chain/jobs.py
      + 75
      − 58
      View file @ 3c82aba9
      ......@@ -435,56 +435,59 @@ class Job:
      """
      try:
      pat = re.compile(r"(?P<cm_version>CM00[1-2])")
      # return pat.match(identifier).groupdict()
      except:
      pass
      try:
      pat = re.compile(
      r"(?P<cm_version>CM003)"
      + "-PRB(?P<probability>.*)"
      + "-ITER(?P<iterations>.*)"
      )
      # return pat.match(identifier).groupdict()
      except:
      pass
      try:
      pat = re.compile(
      r"(?P<cm_version>CM004)"
      + "-CSH(?P<cld_shad>.*)"
      + "-CMP(?P<cld_med_prob>.*)"
      + "-CHP(?P<cld_hi_prob>.*)"
      + "-TCI(?P<thin_cir>.*)"
      + "-ITER(?P<iterations>.*)"
      )
      # return pat.match(identifier).groupdict()
      except:
      pass
      try:
      returned_val = pat.match(identifier).groupdict()
      if "probability" not in returned_val:
      returned_val["probability"] = 1
      if "iterations" not in returned_val:
      returned_val["iterations"] = 5
      if "cld_shad" not in returned_val:
      returned_val["cld_shad"] = True
      if "cld_med_prob" not in returned_val:
      returned_val["cld_med_prob"] = True
      if "cld_hi_prob" not in returned_val:
      returned_val["cld_hi_prob"] = True
      if "thin_cir" not in returned_val:
      returned_val["thin_cir"] = True
      except:
      returned_val = {
      "cm_version": "cm001",
      "probability": 1,
      "iterations": 5,
      "cld_shad": True,
      "cld_med_prob": True,
      "cld_hi_prob": True,
      "thin_cir": True,
      }
      return returned_val
      try:
      pat = re.compile(
      r"(?P<cm_version>CM003)"
      + "-PRB(?P<probability>.*)"
      + "-ITER(?P<iterations>.*)"
      )
      returned_val = pat.match(identifier).groupdict()
      except:
      try:
      pat = re.compile(
      r"(?P<cm_version>CM004)"
      + "-CSH(?P<cld_shad>.*)"
      + "-CMP(?P<cld_med_prob>.*)"
      + "-CHP(?P<cld_hi_prob>.*)"
      + "-TCI(?P<thin_cir>.*)"
      + "-ITER(?P<iterations>.*)"
      )
      returned_val = pat.match(identifier).groupdict()
      except:
      returned_val = {
      "cm_version": "cm001",
      "probability": 1,
      "iterations": 5,
      "cld_shad": True,
      "cld_med_prob": True,
      "cld_hi_prob": True,
      "thin_cir": True,
      }
      if "probability" not in returned_val:
      returned_val["probability"] = 1
      if "iterations" not in returned_val:
      returned_val["iterations"] = 5
      if "cld_shad" not in returned_val:
      returned_val["cld_shad"] = True
      if "cld_med_prob" not in returned_val:
      returned_val["cld_med_prob"] = True
      if "cld_hi_prob" not in returned_val:
      returned_val["cld_hi_prob"] = True
      if "thin_cir" not in returned_val:
      returned_val["thin_cir"] = True
      returned_val["probability"] = int(returned_val["probability"])
      returned_val["iterations"] = int(returned_val["iterations"])
      returned_val["cld_shad"] = returned_val["cld_shad"] in ["1", 1, True, "True"]
      returned_val["cld_med_prob"] = returned_val["cld_med_prob"] in ["1", 1, True, "True"]
      returned_val["cld_hi_prob"] = returned_val["cld_hi_prob"] in ["1", 1, True, "True"]
      returned_val["thin_cir"] = returned_val["thin_cir"] in ["1", 1, True, "True"]
      return returned_val
      def run(
      self,
      ......@@ -662,7 +665,15 @@ class Job:
      t = Tile(row.tile)
      l2a_to_process = [
      p.identifier
      for p in t.cloudmasks_missing().filter_dates(
      for p in t.cloudmasks_missing(
      cm_version = cloudmask["cm_version"].lower(),
      probability = cloudmask["probability"],
      iterations = cloudmask["iterations"],
      cld_shad = cloudmask["cld_shad"],
      cld_med_prob = cloudmask["cld_med_prob"],
      cld_hi_prob = cloudmask["cld_hi_prob"],
      thin_cir = cloudmask["thin_cir"],
      ).filter_dates(
      date_min=row.date_min, date_max=row.date_max
      )
      ]
      ......@@ -702,11 +713,9 @@ class Job:
      "Cloudmask computing process list ({} l2a products):\n".format(
      len(cld_l2a_process_list), )
      )
      if cld_l2a_process_list:
      for cld in cld_l2a_process_list:
      f.write("{}\n\n".format(cld))
      else:
      f.write("\n")
      for cld in cld_l2a_process_list:
      f.write("{}\n".format(cld))
      f.write("\n")
      f.flush()
      ......@@ -735,6 +744,13 @@ class Job:
      for p in t.missing_indices(
      indice=i,
      nodata_clouds=nodata_clouds,
      cm_version = cloudmask["cm_version"].lower(),
      probability = cloudmask["probability"],
      iterations = cloudmask["iterations"],
      cld_shad = cloudmask["cld_shad"],
      cld_med_prob = cloudmask["cld_med_prob"],
      cld_hi_prob = cloudmask["cld_hi_prob"],
      thin_cir = cloudmask["thin_cir"],
      ).filter_dates(
      date_min=row.date_min, date_max=row.date_max
      )
      ......@@ -761,12 +777,13 @@ class Job:
      )
      )
      if self.logs:
      f.write(
      "{}: adding - {}: {} l2a products\n".format(
      row.tile, i, len(l2a_to_process)
      if len(l2a_to_process) > 0:
      f.write(
      "{}: adding - {}: {} l2a products\n".format(
      row.tile, i, len(l2a_to_process)
      )
      )
      )
      f.flush()
      f.flush()
      logger.info(
      "Indice computing process list ({} l2a products): {}".format(
      len(indices_l2a_process_list), indices_l2a_process_list
      ......
      0% or .
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment