moved monthly summary to sen2change, updated library function to get size of...

moved monthly summary to sen2change, updated library function to get size of l1c products to archive with the size=True parameter

sen2chain/__init__.py

@@ -28,10 +28,9 @@ from .indices import IndicesCollection
 from .download_and_process import DownloadAndProcess
 from .time_series import TimeSeries
 from .automatization import Automatization
-from .utils import format_word, grouper, datetime_to_str, str_to_datetime
+from .utils import format_word, grouper, datetime_to_str, str_to_datetime, human_size_decimal, human_size
 from .geo_utils import serialise_tiles_index, get_processed_indices_vect
 from .multi_processing import l2a_multiprocessing, cldidx_multiprocessing, cld_multiprocessing, idx_multiprocessing
-from .monthly_summary import MonthlySummary
 
 __version__ = "0.1.0"
 __author__ = "Jérémy Commins <jebins@openmailbox.org> & Impact <pascal.mouquet@ird.fr>"

sen2chain/library.py

@@ -16,7 +16,7 @@ from typing import List, Dict
 from .config import Config, SHARED_DATA
 from .products import L1cProduct
 from .tiles import Tile
-
+from .utils import human_size, human_size_decimal
 
 s2_tiles_index = SHARED_DATA.get("tiles_index")
 
@@ -77,16 +77,23 @@ class Library:
             
     def archive_l1c(self,
                     archive_list: list = [],
+                    size: bool = False,
                     ):
-        if archive_list:
-            for t in archive_list:
-                try:
-                    til = Tile(t)
-                    til.archive_l1c()
-                except:
-                    pass
-        else:
-            logger.info("Please specify a tile list to archive")
+        total_size = 0 
+        if not archive_list:
+            archive_list = self.l1c
+        for t in archive_list:
+            try:
+                logger.info(t)
+                til = Tile(t)
+                size_tile = til.archive_l1c(size = size)
+                if size:
+                    total_size += size_tile
+                    logger.info("Total size: {}".format(human_size_decimal(total_size)))
+            except:
+                pass
+        #~ else:
+            #~ logger.info("Please specify a tile list to archive")
         
     def update_latest_ql(self):
         """

sen2chain/monthly_summary.py

-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-Module to compute monthly summaries for indices
-"""
-
-import logging
-import datetime
-import pandas as pd
-from pathlib import Path
-import rasterio
-import numpy as np
-
-from .tiles import Tile
-from .config import Config
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-
-class MonthlySummary:
-    """Class for managing monthly summaries.
-
-    :param tile: tile number "40KCB"
-    :param indice: "NDVI"
-    :start_date: "2020-01-15"
-    :end_date: "2020-01-19"
-
-    Usage:
-        >>> MonthlySummary("40KCB", "ndvi", "2019-01-01", "2019-10-31")
-    """
-    
-    def __init__(self, 
-                 tile: str = None,
-                 indice: str = None,
-                 start_date: str = "2015-01-01",
-                 stop_date: str = "2100-01-31",
-                 ):
-        try:
-            self.tile = Tile(tile)
-            self.indice = indice.lower()
-            self.datetime_start = max(datetime.datetime.strptime("2015-01-01", '%Y-%m-%d'), datetime.datetime.strptime(start_date, '%Y-%m-%d'))
-            self.datetime_stop = min(datetime.datetime.strptime(stop_date, '%Y-%m-%d'), datetime.datetime.now())
-        except:
-            logger.info("Mauvaise initialisation")
-            logger.info("Usage: MonthlySummary('40KCB', 'ndvi', '2019-01-01', '2019-05-20')")
-            return
-        
-        _temporal_summary_path = Path(Config().get("temporal_summaries_path"))
-        logger.info(_temporal_summary_path)
-        
-        self.start_list = sorted(set([self.datetime_start.strftime("%Y-%m-%d")] + pd.date_range(self.datetime_start, self.datetime_stop, freq='MS').strftime("%Y-%m-%d").to_list()))
-        self.stop_list = sorted(set(pd.date_range(self.datetime_start, self.datetime_stop, freq='M').strftime("%Y-%m-%d").to_list() + [self.datetime_stop.strftime("%Y-%m-%d")]))
-        #~ logger.info(self.start_list)
-        #~ logger.info(self.stop_list)
-        
-        for index, (start, stop) in enumerate(zip(self.start_list, self.stop_list)):
-            
-            process_list = getattr(self.tile, self.indice).masks.filter_dates(start, stop)
-            logger.info("Compiling {} ({} products)".format(start[:-3], len(process_list)))
-            
-            count=0
-            summary_count = None
-            coef_count = None
-            
-            for product, ccover in ([p.identifier, p.cloud_cover] for p in process_list):
-                count += 1
-                logger.info("{} - {} - {}".format(product, ccover, (1-ccover/100)))
-                #~ logger.info(ccover)
-                product_path = self.tile._paths['indices'][self.indice] / product[:(-12-len(self.indice))] / product
-                #~ logger.info(product_path)
-                
-                with rasterio.open(product_path) as prod_src:
-                    prod_profile = prod_src.profile
-                    prod = prod_src.read(1).astype(np.int16)
-                    try:
-                        summary_count += np.where(prod != 16383, prod * (1-ccover/100), 0).astype(np.int32)
-                        #~ summary_count += prod * (1-ccover/100)
-                        #~ coef_count += (1-ccover/100)
-                        coef_count += np.where(prod != 16383, (1-ccover/100), 0).astype(np.float)
-                    except:
-                        summary_count = np.where(prod != 16383, prod * (1-ccover/100), 0).astype(np.int32)
-                        coef_count = np.where(prod != 16383, (1-ccover/100), 0).astype(np.float)
-                    #~ logger.info(summary_count)
-                    #~ logger.info(coef_count)
-                    
-            if count:
-                logger.info("Compiled {} images".format(count))
-                prod_summary = np.where(coef_count != 0, (summary_count / coef_count).astype(np.int16), 32767)
-                prod_profile.update(driver="Gtiff",
-                                      compress="NONE",
-                                      tiled=False,
-                                      dtype=np.int16,
-                                      nodata=32767,
-                                      transform=prod_src.transform,
-                                      count=1)
-                prod_profile.pop('tiled', None)
-                #~ prod_profile.pop('nodata', None)
-            
-                outpath_tif = _temporal_summary_path / \
-                                self.indice.upper() / \
-                                self.tile.name / \
-                                (self.tile.name + "_" + self.indice.upper() + "_MONTHLY_" + start[:-3] + '.tif')
-                outpath_tif.parent.mkdir(parents = True, exist_ok = True)
-                
-                with rasterio.Env(GDAL_CACHEMAX=512) as env:
-                    with rasterio.open(str(outpath_tif), "w", **prod_profile) as dst:
-                        dst.write(prod_summary, 1)

sen2chain/tiles.py

@@ -494,7 +494,8 @@ class Tile:
                                 nb_rm += 1
         return {"identified_problems": nb_id, "removed_problems": nb_rm}
             
-    def archive_l1c(self):
+    def archive_l1c(self, 
+                    size: bool = False,):
         """
         Check and move l1c products to archive folder
         
@@ -512,18 +513,27 @@ class Tile:
                                 "cloud_cover": self._products["l1c"][prod].cloud_cover}
         if prod_list:
             count = 0
+            total_size = 0 
             for prod in prod_list:
                 l1c = L1cProduct(prod.identifier)
                 if not l1c.path.is_symlink():
                     count += 1
-                    move_path = l1c_archive_path / l1c.tile / l1c.path.name
-                    logger.info("archiving {}".format(l1c.identifier))
-                    move_path.parent.mkdir(exist_ok=True)
-                    #~ shutil.move(str(l1c.path), str(move_path.parent))
-                    distutils.dir_util.copy_tree(str(l1c.path), str(move_path))
-                    distutils.dir_util.remove_tree(str(l1c.path))
-                    l1c.path.symlink_to(move_path, target_is_directory = True)
+                    if size:
+                        total_size += getFolderSize(str(l1c.path))
+                        #~ logger.info("{}: {}".format(prod, human_size(getFolderSize(str(l1c.path)))))
+                    else:
+                        move_path = l1c_archive_path / l1c.tile / l1c.path.name
+                        logger.info("archiving {}".format(l1c.identifier))
+                        move_path.parent.mkdir(exist_ok=True)
+                        #~ shutil.move(str(l1c.path), str(move_path.parent))
+                        distutils.dir_util.copy_tree(str(l1c.path), str(move_path))
+                        distutils.dir_util.remove_tree(str(l1c.path))
+                        l1c.path.symlink_to(move_path, target_is_directory = True)
+                    
             logger.info("{} products archived".format(count))
+            if size:
+                logger.info("Total size to move: {}".format(human_size(total_size)))
+                return total_size
         if not count:
             logger.info("No L1C products to archive")
                 

sen2chain/utils.py

@@ -75,7 +75,15 @@ def datetime_to_str(date: datetime, date_format: str) -> str:
 
 def human_size(bytes, units=['B','KB','MB','GB','TB', 'PB', 'EB']):
     """ Returns a human readable string reprentation of bytes"""
-    return str(bytes) + units[0] if bytes < 1024 else human_size(bytes>>10, units[1:])
+    return str(bytes) + units[0] if bytes < 1024.0 else human_size(bytes>>10, units[1:])
+    
+def human_size_decimal(size, decimal_places=2):
+    for unit in ['B','KB','MB','GB','TB', 'PB', 'EB']:
+        if size < 1024.0:
+            break
+        size /= 1024.0
+    return "{}{}".format(format(size, '.' + str(decimal_places) + 'f'), unit)
+    #~ return f"{size:.{decimal_places}f}{unit}"
 
 def getFolderSize(folder, follow_symlinks = False):
     total_size = os.path.getsize(folder)