From 04855bc1e78e4960c4679a53c5d9fcba7347a0c0 Mon Sep 17 00:00:00 2001
From: ptresson <paul.tresson@ird.fr>
Date: Wed, 12 Mar 2025 19:14:59 +0100
Subject: [PATCH] cleanup merge function, explicitely close datasets. May solve
#45 more properly
---
encoder.py | 5 +++--
utils/geo.py | 16 +++++++---------
2 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/encoder.py b/encoder.py
index 1f000a5..810b632 100644
--- a/encoder.py
+++ b/encoder.py
@@ -618,7 +618,8 @@ class EncoderAlgorithm(IAMAPAlgorithm):
]
all_tiles = [f for f in all_tiles if not f.startswith("merged")]
- dst_path = Path(os.path.join(self.output_subdir, f"{current}_merged_tmp.tif"))
+ # dst_path = Path(os.path.join(self.output_subdir, f"{current}_merged_tmp.tif"))
+ dst_path = Path(os.path.join(self.output_subdir, f"merged_tmp.tif"))
try:
merge_tiles(
@@ -631,7 +632,7 @@ class EncoderAlgorithm(IAMAPAlgorithm):
# overwritting merged_tmp.tif may be impossible in windows (e.g. if an antivirus is analysing the newly created data)
# then, merging and cleaning is impossible
except Exception as e :
- feedback.pushWarning(f"Unable to clean temporary files ! Try to delete them latter at {self.output_subdir}")
+ feedback.pushWarning(f"Unable to clean temporary files ! Try to delete them latter at {self.output_subdir}\n{e}")
self.all_encoding_done = True
diff --git a/utils/geo.py b/utils/geo.py
index 21202d3..e21aab4 100644
--- a/utils/geo.py
+++ b/utils/geo.py
@@ -44,9 +44,8 @@ def merge_tiles(
cf. https://amanbagrecha.github.io/posts/2022-07-31-merge-rasters-the-modern-way-using-python/index.html
"""
- # file_handler = [rasterio.open(ds) for ds in tiles]
- # extents = [ds.bounds for ds in file_handler]
- extents = get_extents(tiles)
+ file_handler = [rasterio.open(ds) for ds in tiles]
+ extents = [ds.bounds for ds in file_handler]
# Extract individual bounds
lefts, bottoms, rights, tops = zip(*extents)
union_extent = (
@@ -59,20 +58,19 @@ def merge_tiles(
if method == "average":
method = custom_method_avg
- # memfile = MemoryFile()
merge(
- # sources=file_handler, # list of dataset objects opened in 'r' mode
- sources=tiles, # try without rasterio.open
+ sources=file_handler, # list of dataset objects opened in 'r' mode
bounds=union_extent, # tuple
nodata=nodata, # float
dtype=dtype, # dtype
- # resampling=Resampling.nearest,
method=method, # strategy to combine overlapping rasters
- # dst_path=memfile.name, # str or PathLike to save raster
dst_path=dst_path,
- # dst_kwds={'blockysize':512, 'blockxsize':512} # Dictionary
)
+ # close datasets
+ for ds in file_handler:
+ ds.close()
+
def get_mean_sd_by_band(path, force_compute=True, ignore_zeros=True, subset=1_000):
"""
--
GitLab