{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import xarray as xr\n",
"from dask.distributed import Client\n",
"import dask.array as da\n",
"import os\n",
"import numpy as np\n",
"from typing import List, Tuple, Union\n",
"import warnings\n",
"import pandas as pd\n",
"import netCDF4 as nc\n",
"from parameters.params_samir_class import samir_parameters\n",
"from config.config import config\n",
"from time import time"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"def rasterize_samir_parameters(csv_param_file: str, empty_dataset: xr.Dataset, land_cover_raster: str, chunk_size: dict) -> Tuple[xr.Dataset, dict]:\n",
" \"\"\"\n",
" Creates a raster `xarray` dataset from the csv parameter file, the land cover raster and an empty dataset\n",
" that contains the right structure (emptied ndvi dataset for example). For each parameter, the function loops\n",
" on land cover classes to fill the raster.\n",
"\n",
" ## Arguments\n",
" 1. csv_param_file: `str`\n",
" path to csv paramter file\n",
" 2. empty_dataset: `xr.Dataset`\n",
" empty dataset that contains the right structure (emptied ndvi dataset for example).\n",
" 3. land_cover_raster: `str`\n",
" path to land cover netcdf raster\n",
" 4. chunk_size: `dict`\n",
" chunk_size for dask computation\n",
"\n",
" ## Returns\n",
" 1. parameter_dataset: `xr.Dataset`\n",
" the dataset containing all the rasterized Parameters\n",
" 2. scale_factor: `dict`\n",
" dictionnary containing the scale factors for each parameter\n",
" \"\"\"\n",
" \n",
" # Load samir params into an object\n",
" table_param = samir_parameters(csv_param_file)\n",
" \n",
" # Set general variables\n",
" class_count = table_param.table.shape[1] - 2 # remove dtype and default columns\n",
" \n",
" # Open land cover raster\n",
" land_cover = xr.open_dataarray(land_cover_raster, chunks = chunk_size)\n",
" \n",
" # Create dataset\n",
" parameter_dataset = empty_dataset.copy(deep = True)\n",
" \n",
" # Create dictionnary containing the scale factors\n",
" scale_factor = {}\n",
" \n",
" # Loop on samir parameters and create \n",
" for parameter in table_param.table.index[1:]:\n",
" \n",
" # Create new variable and set attributes\n",
" parameter_dataset[parameter] = land_cover.copy(deep = True).astype('f4')\n",
" parameter_dataset[parameter].attrs['name'] = parameter\n",
" parameter_dataset[parameter].attrs['description'] = 'cf SAMIR Doc for detail'\n",
" parameter_dataset[parameter].attrs['scale factor'] = str(table_param.table.loc[table_param.table.index == parameter]['scale_factor'].values[0])\n",
" \n",
" # Assigne value in dictionnary\n",
" scale_factor[parameter] = 1/int(table_param.table.loc[table_param.table.index == parameter]['scale_factor'].values[0])\n",
" \n",
" # Loop on classes to set parameter values for each class\n",
" for class_val, class_name in zip(range(1, class_count + 1), table_param.table.columns[2:]):\n",
" \n",
" # Parameter values are multiplied by the scale factor in order to store all values as int16 types\n",
" # These values are then rounded to make sure there isn't any decimal point issues when casting the values to int16\n",
" parameter_dataset[parameter].values = np.where(parameter_dataset[parameter].values == class_val, round(table_param.table.loc[table_param.table.index == parameter][class_name].values[0]*table_param.table.loc[table_param.table.index == parameter]['scale_factor'].values[0]), parameter_dataset[parameter].values).astype('f4')\n",
" \n",
" # Return dataset converted to 'int16' data type to reduce memory usage\n",
" # and scale_factor dictionnary for later conversion\n",
" return parameter_dataset, scale_factor\n",
"\n",
"\n",
"def setup_time_loop(calculation_variables_t1: List[str], calculation_variables_t2: List[str], empty_dataset: xr.Dataset) -> Tuple[xr.Dataset, xr.Dataset]:\n",
" \"\"\"\n",
" Creates two temporary `xarray Datasets` that will be used in the SAMIR time loop.\n",
" `variables_t1` corresponds to the variables for the previous day and `variables_t2`\n",
" corresponds to the variables for the current day. After each loop, `variables_t1`\n",
" takes the value of `variables_t2` for the corresponding variables.\n",
"\n",
" ## Arguments\n",
" 1. calculation_variables_t1: `List[str]`\n",
" list of strings containing the variable names\n",
" for the previous day dataset\n",
" 2. calculation_variables_t2: `List[str]`\n",
" list of strings containing the variable names\n",
" for the current day dataset\n",
" 3. empty_dataset: `xr.Dataset`\n",
" empty dataset that contains the right structure\n",
"\n",
" ## Returns\n",
" 1. variables_t1: `xr.Dataset`\n",
" output dataset for previous day\n",
" 2. variables_t2: `xr.Dataset`\n",
" output dataset for current day\n",
" \"\"\"\n",
" \n",
" # Create new dataset\n",
" variables_t1 = empty_dataset.copy(deep = True)\n",
" \n",
" # Create empty DataArray for each variable\n",
" for variable in calculation_variables_t1:\n",
" \n",
" # Assign new empty DataArray\n",
" variables_t1[variable] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'float32'))\n",
" variables_t1[variable].attrs['name'] = variable # set name in attributes\n",
" \n",
" # Create new dataset\n",
" variables_t2 = empty_dataset.copy(deep = True)\n",
" \n",
" # Create empty DataArray for each variable\n",
" for variable in calculation_variables_t2:\n",
" \n",
" # Assign new empty DataArray\n",
" variables_t2[variable] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'float32'))\n",
" variables_t2[variable].attrs['name'] = variable # set name in attributes\n",
" \n",
" return variables_t1, variables_t2\n",
"\n",
"\n",
"def prepare_outputs(empty_dataset: xr.Dataset, additional_outputs: List[str] = None) -> xr.Dataset:\n",
" \"\"\"\n",
" Creates the `xarray Dataset` containing the outputs of the SAMIR model that will be saved.\n",
" Additional variables can be saved by adding their names to the `additional_outputs` list.\n",
"\n",
" ## Arguments\n",
" 1. empty_dataset: `xr.Dataset`\n",
" empty dataset that contains the right structure\n",
" 2. additional_outputs: `List[str]`\n",
" list of additional variable names to be saved\n",
"\n",
" ## Returns\n",
" 1. model_outputs: `xr.Dataset`\n",
" model outputs to be saved\n",
" \"\"\"\n",
" \n",
" # Evaporation and Transpiraion\n",
" model_outputs = empty_dataset.copy(deep = True)\n",
" \n",
" model_outputs['E'] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" model_outputs['E'].attrs['units'] = 'mm'\n",
" model_outputs['E'].attrs['standard_name'] = 'Evaporation'\n",
" model_outputs['E'].attrs['description'] = 'Accumulated daily evaporation in milimeters'\n",
" model_outputs['E'].attrs['scale factor'] = '1'\n",
" \n",
" model_outputs['Tr'] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" model_outputs['Tr'].attrs['units'] = 'mm'\n",
" model_outputs['Tr'].attrs['standard_name'] = 'Transpiration'\n",
" model_outputs['Tr'].attrs['description'] = 'Accumulated daily plant transpiration in milimeters'\n",
" model_outputs['Tr'].attrs['scale factor'] = '1'\n",
" \n",
" # Soil Water Content\n",
" model_outputs['SWCe'] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" model_outputs['SWCe'].attrs['units'] = 'mm'\n",
" model_outputs['SWCe'].attrs['standard_name'] = 'Soil Water Content of the evaporative zone'\n",
" model_outputs['SWCe'].attrs['description'] = 'Soil water content of the evaporative zone in milimeters'\n",
" model_outputs['SWCe'].attrs['scale factor'] = '1'\n",
" \n",
" model_outputs['SWCr'] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" model_outputs['SWCr'].attrs['units'] = 'mm'\n",
" model_outputs['SWCr'].attrs['standard_name'] = 'Soil Water Content of the root zone'\n",
" model_outputs['SWCr'].attrs['description'] = 'Soil water content of the root zone in milimeters'\n",
" model_outputs['SWCr'].attrs['scale factor'] = '1'\n",
" \n",
" # Irrigation\n",
" model_outputs['Irr'] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" model_outputs['Irr'].attrs['units'] = 'mm'\n",
" model_outputs['Irr'].attrs['standard_name'] = 'Irrigation'\n",
" model_outputs['Irr'].attrs['description'] = 'Simulated daily irrigation in milimeters'\n",
" model_outputs['Irr'].attrs['scale factor'] = '1'\n",
" \n",
" # Deep Percolation\n",
" model_outputs['DP'] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" model_outputs['DP'].attrs['units'] = 'mm'\n",
" model_outputs['DP'].attrs['standard_name'] = 'Deep Percolation'\n",
" model_outputs['DP'].attrs['description'] = 'Simulated daily Deep Percolation in milimeters'\n",
" model_outputs['DP'].attrs['scale factor'] = '1'\n",
" \n",
" if additional_outputs:\n",
" for var in additional_outputs:\n",
" model_outputs[var] = (empty_dataset.dims, np.zeros(tuple(empty_dataset.dims[d] for d in list(empty_dataset.dims)), dtype = 'int16'))\n",
" \n",
" return model_outputs\n",
"\n",
"\n",
"def xr_maximum(ds: xr.DataArray, value: Union[xr.DataArray, float, int]) -> xr.DataArray:\n",
" \"\"\"\n",
" Equivalent of `numpy.maximum(ds, value)` for xarray DataArrays\n",
"\n",
" ## Arguments\n",
" 1. ds: `xr.DataArray`\n",
" datarray to compare\n",
" 2. value: `Union[xr.DataArray, float, int]`\n",
" value (scalar or dataarray) to compare\n",
"\n",
" ## Returns\n",
" 1. output: `xr.DataArray`\n",
" resulting dataarray with maximum value element-wise\n",
" \"\"\"\n",
" return xr.where(ds <= value, value, ds, keep_attrs = True)\n",
"\n",
"\n",
"def xr_minimum(ds: xr.DataArray, value: Union[xr.DataArray, float, int]) -> xr.DataArray:\n",
" \"\"\"\n",
" Equivalent of `numpy.minimum(ds, value)` for xarray DataArrays\n",
"\n",
" ## Arguments\n",
" 1. ds: `xr.DataArray`\n",
" datarray to compare\n",
" 2. value: `Union[xr.DataArray, float, int]`\n",
" value (scalar or dataarray) to compare\n",
"\n",
" ## Returns\n",
" 1. output: `xr.DataArray`\n",
" resulting dataarray with minimum value element-wise\n",
" \"\"\"\n",
" return xr.where(ds >= value, value, ds, keep_attrs = True)\n",
"\n",
"\n",
"def calculate_diff_re(TAW: np.ndarray, Dr: np.ndarray, Zr: np.ndarray, RUE: np.ndarray, De: np.ndarray, FCov: np.ndarray, Ze_: np.ndarray, DiffE_: np.ndarray, scale_dict: dict) -> np.ndarray:\n",
" \"\"\"\n",
" Calculates the diffusion between the top soil layer and the root layer.\n",
"\n",
" ## Arguments\n",
" 1. TAW: `np.ndarray`\n",
" water capacity of root layer\n",
" 2. Dr: `np.ndarray`\n",
" depletion of root layer\n",
" 3. Zr: `np.ndarray`\n",
" height of root layer\n",
" 4. RUE: `np.ndarray`\n",
" total available surface water\n",
" 5. De: `np.ndarray`\n",
" depletion of the evaporative layer\n",
" 6. FCov: `np.ndarray`\n",
" fraction cover of plants\n",
" 7. Ze_: `np.ndarray`\n",
" height of evaporative layer (paramter)\n",
" 8. DiffE_: `np.ndarray`\n",
" diffusion coefficient between evaporative\n",
" and root layers (unitless, parameter)\n",
" 9. scale_dict: `dict`\n",
" dictionnary containing the scale factors for\n",
" the rasterized parameters\n",
"\n",
" ## Returns\n",
" 1. diff_re: `np.ndarray`\n",
" the diffusion between the top soil layer and\n",
" the root layer\n",
" \"\"\"\n",
" \n",
" # Temporary variables to make calculation easier to read\n",
" tmp1 = (((TAW - Dr) / Zr - (RUE - De) / (scale_dict['Ze'] * Ze_)) / FCov) * (scale_dict['DiffE'] * DiffE_)\n",
" tmp2 = ((TAW * scale_dict['Ze'] * Ze_) - (RUE - De - Dr) * Zr) / (Zr + scale_dict['Ze'] * Ze_) - Dr\n",
" \n",
" # Calculate diffusion according to SAMIR equation\n",
" diff_re = np.where(tmp1 < 0, np.maximum(tmp1, tmp2), np.minimum(tmp1, tmp2))\n",
"\n",
" # Return zero values where the 'DiffE' parameter is equal to 0\n",
" return np.where(DiffE_ == 0, 0, diff_re)\n",
"\n",
"\n",
"def calculate_diff_dr(TAW: np.ndarray, TDW: np.ndarray, Dr: np.ndarray, Zr: np.ndarray, Dd: np.ndarray, FCov: np.ndarray, Zsoil_: np.ndarray, DiffR_: np.ndarray, scale_dict: dict) -> np.ndarray:\n",
" \"\"\"\n",
" Calculates the diffusion between the root layer and the deep layer.\n",
"\n",
" ## Arguments\n",
" 1. TAW: `np.ndarray`\n",
" water capacity of root layer\n",
" 2. TDW: `np.ndarray`\n",
" water capacity of deep layer\n",
" 3. Dr: `np.ndarray`\n",
" depletion of root layer\n",
" 4. Zr: `np.ndarray`\n",
" height of root layer\n",
" 5. Dd: `np.ndarray`\n",
" depletion of deep layer\n",
" 6. FCov: `np.ndarray`\n",
" fraction cover of plants\n",
" 7. Zsoil_: `np.ndarray`\n",
" total height of soil (paramter)\n",
" 8. DiffR_: `np.ndarray`\n",
" Diffusion coefficient between root\n",
" and deep layers (unitless, parameter)\n",
" 9. scale_dict: `dict`\n",
" dictionnary containing the scale factors for\n",
" the rasterized parameters\n",
"\n",
" ## Returns\n",
" 1. diff_dr: `np.ndarray`\n",
" the diffusion between the root layer and the\n",
" deep layer\n",
" \"\"\"\n",
" \n",
" # Temporary variables to make calculation easier to read\n",
" tmp1 = (((TDW - Dd) / (scale_dict['Zsoil'] * Zsoil_ - Zr) - (TAW - Dr) / Zr) / FCov) * scale_dict['DiffR'] * DiffR_\n",
" tmp2 = (TDW *Zr - (TAW - Dr - Dd) * (scale_dict['Zsoil'] * Zsoil_ - Zr)) / (scale_dict['Zsoil'] * Zsoil_) - Dd\n",
" \n",
" # Calculate diffusion according to SAMIR equation\n",
" diff_dr = np.where(tmp1 < 0, np.maximum(tmp1, tmp2), np.minimum(tmp1, tmp2))\n",
" \n",
" # Return zero values where the 'DiffR' parameter is equal to 0\n",
" return np.where(DiffR_ == 0, 0, diff_dr)\n",
"\n",
"\n",
"def calculate_W(TEW: np.ndarray, Dei: np.ndarray, Dep: np.ndarray, fewi: np.ndarray, fewp: np.ndarray) -> np.ndarray:\n",
" \"\"\"\n",
" Calculate W, the weighting factor to split the energy available\n",
" for evaporation depending on the difference in water availability\n",
" in the two evaporation components, ensuring that the larger and\n",
" the wetter, the more the evaporation occurs from that component\n",
"\n",
" ## Arguments\n",
" 1. TEW: `np.ndarray`\n",
" water capacity of evaporative layer\n",
" 2. Dei: `np.ndarray`\n",
" depletion of the evaporative layer\n",
" (irrigation part)\n",
" 3. Dep: `np.ndarray`\n",
" depletion of the evaporative layer\n",
" (precipitation part)\n",
" 4. fewi: `np.ndarray`\n",
" soil fraction which is wetted by irrigation\n",
" and exposed to evaporation\n",
" 5. fewp: `np.ndarray`\n",
" soil fraction which is wetted by precipitation\n",
" and exposed to evaporation\n",
"\n",
" ## Returns\n",
" 1. W: `np.ndarray`\n",
" weighting factor W\n",
" \"\"\"\n",
" \n",
" # Temporary variables to make calculation easier to read\n",
" tmp = fewi * (TEW - Dei)\n",
" \n",
" # Calculate the weighting factor to split the energy available for evaporation\n",
" W = 1 / (1 + (fewp * (TEW - Dep) / tmp ))\n",
"\n",
" # Return W \n",
" return np.where(tmp > 0, W, 0)\n",
"\n",
"\n",
"def calculate_Kr(TEW: np.ndarray, De: np.ndarray, REW_: np.ndarray, scale_dict: dict) -> np.ndarray:\n",
" \"\"\"\n",
" calculates of the reduction coefficient for evaporation dependent \n",
" on the amount of water in the soil using the FAO-56 method\n",
"\n",
" ## Arguments\n",
" 1. TEW: `np.ndarray`\n",
" water capacity of evaporative layer\n",
" 2. De: `np.ndarray`\n",
" depletion of evaporative layer\n",
" 3. REW_: `np.ndarray`\n",
" readily evaporable water\n",
" 4. scale_dict: `dict`\n",
" dictionnary containing the scale factors for\n",
" the rasterized parameters\n",
"\n",
" ## Returns\n",
" 1. Kr: `np.ndarray`\n",
" Kr coefficient\n",
" \"\"\"\n",
" \n",
" # Formula for calculating Kr\n",
" Kr = (TEW - De) / (TEW - scale_dict['REW'] * REW_)\n",
" \n",
" # Return Kr\n",
" return np.maximum(0, np.minimum(Kr, 1))\n",
"\n",
"\n",
"def update_Dr(TAW: np.ndarray, TDW: np.ndarray, Zr: np.ndarray, TAW0: np.ndarray, TDW0: np.ndarray, Dr0: np.ndarray, Dd0: np.ndarray, Zr0: np.ndarray) -> np.ndarray:\n",
" \"\"\"\n",
" Return the updated depletion for the root layer\n",
"\n",
" ## Arguments\n",
" 1. TAW: `np.ndarray`\n",
" water capacity of root layer for current day\n",
" 2. TDW: `np.ndarray`\n",
" water capacity of deep layer for current day\n",
" 3. Zr: `np.ndarray`\n",
" root layer height for current day\n",
" 4. TAW0: `np.ndarray`\n",
" water capacity of root layer for previous day\n",
" 5. TDW0: `np.ndarray`\n",
" water capacity of deep layer for previous day\n",
" 6. Dr0: `np.ndarray`\n",
" depletion of the root layer for previous day\n",
" 7. Dd0: `np.ndarray`\n",
" depletion of the deep laye for previous day\n",
" 8. Zr0: `np.ndarray`\n",
" root layer height for previous day\n",
"\n",
" ## Returns\n",
" 1. output: `np.ndarray`\n",
" updated depletion for the root layer\n",
" \"\"\"\n",
" \n",
" # Temporary variables to make calculation easier to read\n",
" tmp1 = np.maximum(Dr0 + Dd0 * (TAW - TAW0) / TDW0, 0)\n",
" tmp2 = np.minimum(Dr0 + Dd0 * (TAW - TAW0) / TDW0, TDW)\n",
"\n",
" # Return updated Dr\n",
" return np.where(Zr > Zr0, tmp1, tmp2)\n",
"\n",
"\n",
"def update_Dd(TAW: np.ndarray, TDW: np.ndarray, Zr: np.ndarray, TAW0: np.ndarray, TDW0: np.ndarray, Dd0: np.ndarray, Zr0: np.ndarray) -> np.ndarray:\n",
" \"\"\"\n",
" Return the updated depletion for the deep layer\n",
"\n",
" ## Arguments\n",
" 1. TAW: `np.ndarray`\n",
" water capacity of root layer for current day\n",
" 2. TDW: `np.ndarray`\n",
" water capacity of deep layer for current day\n",
" 3. TAW0: `np.ndarray`\n",
" water capacity of root layer for previous day\n",
" 5. TDW0: `np.ndarray`\n",
" water capacity of deep layer for previous day\n",
" 6. Dd0: `np.ndarray`\n",
" depletion of the deep laye for previous day\n",
" 7. Zr0: `np.ndarray`\n",
" root layer height for previous day\n",
"\n",
" ## Returns\n",
" 1. output: `np.ndarray`\n",
" updated depletion for the deep layer\n",
" \"\"\"\n",
" \n",
" # Temporary variables to make calculation easier to read\n",
" tmp1 = np.maximum(Dd0 - Dd0 * (TAW - TAW0) / TDW0, 0)\n",
" tmp2 = np.minimum(Dd0 - Dd0 * (TAW - TAW0) / TDW0, TDW)\n",
" \n",
" # Return updated Dd\n",
" return np.where(Zr > Zr0, tmp1, tmp2)\n",
"\n",
"\n",
"def format_duration(timedelta: float) -> None:\n",
" \"\"\"\n",
" Print formatted timedelta in human readable format\n",
" (days, hours, minutes, seconds, microseconds, milliseconds, nanoseconds).\n",
"\n",
" ## Arguments\n",
" timedelta: `float`\n",
" time value in seconds to format\n",
"\n",
" ## Returns\n",
" `None`\n",
" \"\"\"\n",
" \n",
" if timedelta < 0.9e-6:\n",
" print(round(timedelta*1e9, 1), 'ns')\n",
" elif timedelta < 0.9e-3:\n",
" print(round(timedelta*1e6, 1), 'µs')\n",
" elif timedelta < 0.9:\n",
" print(round(timedelta*1e3, 1), 'ms')\n",
" elif timedelta < 60:\n",
" print(round(timedelta, 1), 's')\n",
" elif timedelta < 3.6e3:\n",
" print(round(timedelta//60), 'm', round(timedelta%60, 1), 's')\n",
" elif timedelta < 24*3.6e3:\n",
" print(round((timedelta/3.6e3)//1), 'h', round((timedelta/3.6e3)%1*60//1), 'm', round((timedelta/3.6e3)%1*60%1*60, 1), 's' ) \n",
" elif timedelta < 48*3.6e3:\n",
" print(round((timedelta/(24*3.6e3))//1), 'day,', round(((timedelta/(24*3.6e3))%1*24)//1), 'h,', round((timedelta/(24*3.6e3))%1*24%1*60//1), 'm,', round((timedelta/(24*3.6e3))%1*24%1*60%1*60, 1), 's')\n",
" else:\n",
" print(round((timedelta/(24*3.6e3))//1), 'days,', round(((timedelta/(24*3.6e3))%1*24)//1), 'h,', round((timedelta/(24*3.6e3))%1*24%1*60//1), 'm,', round((timedelta/(24*3.6e3))%1*24%1*60%1*60, 1), 's')\n",
" \n",
" return None\n",
"\n",
"\n",
"def run_samir(json_config_file: str, csv_param_file: str, ndvi_cube_path: str, precip_cube_path: str, ET0_cube_path: str, soil_params_path: str, land_cover_path: str, chunk_size: dict, save_path: str, max_GB: int = 2) -> None:\n",
" \n",
" # warnings.simplefilter(\"error\", category = RuntimeWarning())\n",
" warnings.filterwarnings(\"ignore\", message=\"invalid value encountered in cast\")\n",
" warnings.filterwarnings(\"ignore\", message=\"invalid value encountered in divide\")\n",
" np.errstate(all = 'ignore')\n",
" \n",
" #============ General parameters ============#\n",
" config_params = config(json_config_file)\n",
" calculation_variables_t2 = ['diff_rei', 'diff_rep', 'diff_dr' , 'Dd', 'De', 'Dei', 'Dep', 'Dr', 'FCov', 'Irrig', 'Kcb', 'Kei', 'Kep', 'Ks', 'Kti', 'Ktp', 'RUE', 'TAW', 'TDW', 'TEW', 'Tei', 'Tep', 'W', 'Zr', 'fewi', 'fewp']\n",
" calculation_variables_t1 = ['Dr', 'Dd', 'TAW', 'TDW', 'Zr']\n",
" \n",
" #============ Manage inputs ============#\n",
" # NDVI\n",
" ndvi_cube = xr.open_dataset(ndvi_cube_path, chunks = chunk_size).astype('u1')\n",
" \n",
" # Weather\n",
" ## Open geotiff cubes and rename variables and coordinates\n",
" prec_cube = xr.open_dataset(precip_cube_path, chunks = chunk_size).astype('u2').rename({'band': 'time', 'band_data': 'prec'})\n",
" ET0_cube = xr.open_dataset(ET0_cube_path, chunks = chunk_size).astype('u2').rename({'band': 'time', 'band_data': 'ET0'})\n",
" \n",
" ## Reset times values \n",
" prec_cube['time'] = pd.date_range(start = config_params.start_date, end = config_params.end_date, freq = 'D')\n",
" ET0_cube['time'] = pd.date_range(start = config_params.start_date, end = config_params.end_date, freq = 'D')\n",
" \n",
" ## Remove unwanted attributes\n",
" del prec_cube.prec.attrs['AREA_OR_POINT'], ET0_cube.ET0.attrs['AREA_OR_POINT']\n",
" \n",
" # Soil\n",
" soil_params = xr.open_dataset(soil_params_path, chunks = chunk_size).astype('f4')\n",
" \n",
" # SAMIR Parameters\n",
" param_dataset, scale_factor = rasterize_samir_parameters(csv_param_file, ndvi_cube.drop_vars(['ndvi', 'time']), land_cover_path, chunk_size = chunk_size)\n",
" \n",
" # SAMIR Variables\n",
" variables_t1, variables_t2 = setup_time_loop(calculation_variables_t1, calculation_variables_t2, ndvi_cube.drop_vars(['ndvi', 'time']))\n",
" \n",
" # # Manage loading of data based on disk size of inputs\n",
" # if ndvi_cube.nbytes < max_GB * (1024)**3:\n",
" # ndvi_cube.load()\n",
" \n",
" # if weather_cube.nbytes < max_GB * (1024)**3:\n",
" # weather_cube.load()\n",
"\n",
" #============ Prepare outputs ============#\n",
" model_outputs = prepare_outputs(ndvi_cube.drop_vars(['ndvi']))\n",
" \n",
" # Create encoding dictionnary\n",
" for variable in list(model_outputs.keys()):\n",
" # Write encoding dict\n",
" encoding_dict = {}\n",
" encod = {}\n",
" encod['dtype'] = 'i2'\n",
" encoding_dict[variable] = encod\n",
" \n",
" # Save empty output\n",
" model_outputs.to_netcdf(save_path, encoding = encoding_dict)\n",
" model_outputs.close()\n",
" \n",
" #============ Prepare time iterations ============#\n",
" dates = ndvi_cube.time.values\n",
" \n",
" #============ Create aliases for better readability ============#\n",
" \n",
" # Variables for current day\n",
" # var = da.from_array(dataarray, chunks = (5, 5))\n",
" diff_rei = variables_t2.diff_rei.to_numpy()\n",
" diff_rep = variables_t2.diff_rep.to_numpy()\n",
" diff_dr = variables_t2.diff_dr.to_numpy()\n",
" Dd = variables_t2.Dd.to_numpy()\n",
" De = variables_t2.De.to_numpy()\n",
" Dei = variables_t2.Dei.to_numpy()\n",
" Dep = variables_t2.Dep.to_numpy()\n",
" Dr = variables_t2.Dr.to_numpy()\n",
" FCov = variables_t2.FCov.to_numpy()\n",
" Irrig = variables_t2.Irrig.to_numpy()\n",
" Kcb = variables_t2.Kcb.to_numpy()\n",
" Kei = variables_t2.Kei.to_numpy()\n",
" Kep = variables_t2.Kep.to_numpy()\n",
" Ks = variables_t2.Ks.to_numpy()\n",
" Kti = variables_t2.Kti.to_numpy()\n",
" Ktp = variables_t2.Ktp.to_numpy()\n",
" RUE = variables_t2.RUE.to_numpy()\n",
" TAW = variables_t2.TAW.to_numpy()\n",
" TDW = variables_t2.TDW.to_numpy()\n",
" TEW = variables_t2.TEW.to_numpy()\n",
" Tei = variables_t2.Tei.to_numpy()\n",
" Tep = variables_t2.Tep.to_numpy()\n",
" Zr = variables_t2.Zr.to_numpy()\n",
" W = variables_t2.W.to_numpy()\n",
" fewi = variables_t2.fewi.to_numpy()\n",
" fewp = variables_t2.fewp.to_numpy()\n",
" \n",
" # Variables for previous day\n",
" TAW0 = variables_t1.TAW.to_numpy()\n",
" TDW0 = variables_t1.TDW.to_numpy()\n",
" Dr0 = variables_t1.Dr.to_numpy()\n",
" Dd0 = variables_t1.Dd.to_numpy()\n",
" Zr0 = variables_t1.Zr.to_numpy()\n",
" \n",
" # Parameters\n",
" # Parameters have an underscore (_) behind their name for recognition \n",
" DiffE_ = param_dataset.DiffE.to_numpy()\n",
" DiffR_ = param_dataset.DiffR.to_numpy()\n",
" FW_ = param_dataset.FW.to_numpy()\n",
" Fc_stop_ = param_dataset.Fc_stop.to_numpy()\n",
" FmaxFC_ = param_dataset.FmaxFC.to_numpy()\n",
" Foffset_ = param_dataset.Foffset.to_numpy()\n",
" Fslope_ = param_dataset.Fslope.to_numpy()\n",
" Init_RU_ = param_dataset.Init_RU.to_numpy()\n",
" Irrig_auto_ = param_dataset.Irrig_auto.to_numpy()\n",
" Kcmax_ = param_dataset.Kcmax.to_numpy()\n",
" KmaxKcb_ = param_dataset.KmaxKcb.to_numpy()\n",
" Koffset_ = param_dataset.Koffset.to_numpy()\n",
" Kslope_ = param_dataset.Kslope.to_numpy()\n",
" Lame_max_ = param_dataset.Lame_max.to_numpy()\n",
" REW_ = param_dataset.REW.to_numpy()\n",
" Ze_ = param_dataset.Ze.to_numpy()\n",
" Zsoil_ = param_dataset.Zsoil.to_numpy()\n",
" maxZr_ = param_dataset.maxZr.to_numpy()\n",
" minZr_ = param_dataset.minZr.to_numpy()\n",
" p_ = param_dataset.p.to_numpy()\n",
" \n",
" # scale factors\n",
" # Scale factors have the following name scheme : s_ + parameter_name\n",
" s_DiffE = scale_factor['DiffE']\n",
" s_DiffR = scale_factor['DiffR']\n",
" s_FW = scale_factor['FW']\n",
" s_Fc_stop = scale_factor['Fc_stop']\n",
" s_FmaxFC = scale_factor['FmaxFC']\n",
" s_Foffset = scale_factor['Foffset']\n",
" s_Fslope = scale_factor['Fslope']\n",
" s_Init_RU = scale_factor['Init_RU']\n",
" # s_Irrig_auto = scale_factor['Irrig_auto']\n",
" s_Kcmax = scale_factor['Kcmax']\n",
" s_KmaxKcb = scale_factor['KmaxKcb']\n",
" s_Koffset = scale_factor['Koffset']\n",
" s_Kslope = scale_factor['Kslope']\n",
" s_Lame_max = scale_factor['Lame_max']\n",
" s_REW = scale_factor['REW']\n",
" s_Ze = scale_factor['Ze']\n",
" s_Zsoil = scale_factor['Zsoil']\n",
" s_maxZr = scale_factor['maxZr']\n",
" s_minZr = scale_factor['minZr']\n",
" s_p = scale_factor['p']\n",
" \n",
" # input data\n",
" ndvi = ndvi_cube.ndvi.sel({'time': dates[0]}).to_numpy() / 255\n",
" prec = prec_cube.prec.sel({'time': dates[0]}).to_numpy() / 1000\n",
" ET0 = ET0_cube.ET0.sel({'time': dates[0]}).to_numpy() / 1000\n",
"\n",
" #============ First day initialization ============#\n",
" # Fraction cover\n",
" FCov = s_Fslope * Fslope_ * ndvi + s_Foffset * Foffset_\n",
" FCov = np.minimum(np.maximum(FCov, 0), s_Fc_stop * Fc_stop_)\n",
" \n",
" # Root depth upate\n",
" Zr = s_minZr * minZr_ + (FCov / (s_FmaxFC * FmaxFC_)) * s_maxZr * (maxZr_ - minZr_)\n",
" \n",
" # Water capacities\n",
" TEW = (soil_params.FC.values - soil_params.WP.values/2) * s_Ze * Ze_\n",
" RUE = (soil_params.FC.values - soil_params.WP.values) * s_Ze * Ze_\n",
" TAW = (soil_params.FC.values - soil_params.WP.values) * Zr\n",
" TDW = (soil_params.FC.values - soil_params.WP.values) * (s_Zsoil * Zsoil_ - Zr) # Zd = Zsoil - Zr\n",
" \n",
" # Depletions\n",
" Dei = RUE * (1 - s_Init_RU * Init_RU_)\n",
" Dep = RUE * (1 - s_Init_RU * Init_RU_)\n",
" Dr = TAW * (1 - s_Init_RU * Init_RU_)\n",
" Dd = TDW * (1 - s_Init_RU * Init_RU_)\n",
" \n",
" # Irrigation TODO : find correct method for irrigation\n",
" Irrig = np.minimum(np.maximum(Dr - prec, 0), s_Lame_max * Lame_max_) * Irrig_auto_\n",
" Irrig = np.where(Dr > TAW * s_p * p_, Irrig, 0)\n",
" \n",
" # Kcb\n",
" Kcb = np.minimum(s_Kslope * Kslope_ * ndvi + s_Koffset * Koffset_, s_KmaxKcb * KmaxKcb_)\n",
" \n",
" # Update depletions with rainfall and/or irrigation\n",
" \n",
" ## DP \n",
" # Variable directly written since not used later\n",
" # Dimensions of output dataset : (x, y, time)\n",
" outputs = nc.Dataset(save_path, mode='r+')\n",
" outputs.variables['DP'][:,:,0] = np.round(- np.minimum(Dd + np.minimum(Dr - prec - Irrig, 0), 0) * 1000).astype('int16')\n",
" outputs.close()\n",
"\n",
" # model_outputs.DP.loc[{'time': dates[0]}] = - np.minimum(Dd + np.minimum(Dr - prec - Irrig, 0), 0)\n",
" \n",
" ## De\n",
" Dei = np.minimum(np.maximum(Dei - prec - Irrig / (s_FW * FW_ / 100), 0), TEW)\n",
" Dep = np.minimum(np.maximum(Dep - prec, 0), TEW)\n",
" \n",
" fewi = np.minimum(1 - FCov, (s_FW * FW_ / 100))\n",
" fewp = 1 - FCov - fewi\n",
" \n",
" De = np.divide((Dei * fewi + Dep * fewp), (fewi + fewp))\n",
" De = np.where(np.isfinite(De), De, Dei * (s_FW * FW_ / 100) + Dep * (1 - (s_FW * FW_ / 100)))\n",
"\n",
" ## Dr\n",
" Dr = np.minimum(np.maximum(Dr - prec - Irrig, 0), TAW)\n",
" \n",
" ## Dd\n",
" Dd = np.minimum(np.maximum(Dd + np.minimum(Dr - prec - Irrig, 0), 0), TDW)\n",
" \n",
" # Diffusion coefficients\n",
" diff_rei = calculate_diff_re(TAW, Dr, Zr, RUE, Dei, FCov, Ze_, DiffE_, scale_factor)\n",
" diff_rep = calculate_diff_re(TAW, Dr, Zr, RUE, Dep, FCov, Ze_, DiffE_, scale_factor)\n",
" diff_dr = calculate_diff_dr(TAW, TDW, Dr, Zr, Dd, FCov, Zsoil_, DiffR_, scale_factor) \n",
" \n",
" # Weighing factor W\n",
" W = calculate_W(TEW, Dei, Dep, fewi, fewp)\n",
" \n",
" # Write outputs\n",
" # Variables directly written since not used later\n",
" outputs = nc.Dataset(save_path, mode='r+')\n",
" # Soil water content of evaporative layer\n",
" outputs.variables['SWCe'][:,:,0] = np.round((1 - De/TEW) * 1000).astype('int16')\n",
" # Soil water content of root layer\n",
" outputs.variables['SWCe'][:,:,0] = np.round((1 - Dr/TAW) * 1000).astype('int16')\n",
" outputs.close()\n",
" \n",
" \n",
" # model_outputs.SWCe.loc[{'time': dates[0]}] = 1 - De/TEW\n",
" # model_outputs.SWCr.loc[{'time': dates[0]}] = 1 - Dr/TAW\n",
" \n",
" # Water Stress coefficient\n",
" Ks = np.minimum((TAW - Dr) / (TAW * (1 - s_p * p_)), 1)\n",
" \n",
" # Reduction coefficient for evaporation\n",
" Kei = np.minimum(W * calculate_Kr(TEW, Dei, REW_, scale_factor) * (s_Kcmax * Kcmax_ - Kcb), fewi * s_Kcmax * Kcmax_)\n",
" Kep = np.minimum((1 - W) * calculate_Kr(TEW, Dep, REW_, scale_factor) * (s_Kcmax * Kcmax_ - Kcb), fewp * s_Kcmax * Kcmax_)\n",
" \n",
" # Prepare coefficients for evapotranspiration\n",
" Kti = np.minimum(((s_Ze * Ze_ / Zr)**6) * (1 - Dei / TEW) / np.maximum(1 - Dr / TAW, 0.001), 1)\n",
" Ktp = np.minimum(((s_Ze * Ze_ / Zr)**6) * (1 - Dep / TEW) / np.maximum(1 - Dr / TAW, 0.001), 1)\n",
" Tei = Kti * Ks * Kcb * ET0\n",
" Tep = Ktp * Ks * Kcb * ET0\n",
" \n",
" # Update depletions\n",
" Dei = np.where(fewi > 0, np.minimum(np.maximum(Dei + ET0 * Kei / fewi + Tei - diff_rei, 0), TEW), np.minimum(np.maximum(Dei + Tei - diff_rei, 0), TEW))\n",
" Dep = np.where(fewp > 0, np.minimum(np.maximum(Dep + ET0 * Kep / fewp + Tep - diff_rep, 0), TEW), np.minimum(np.maximum(Dep + Tep - diff_rep, 0), TEW))\n",
" \n",
" De = (Dei * fewi + Dep * fewp) / (fewi + fewp)\n",
" De = np.where(np.isfinite(De), De, Dei * (s_FW * FW_ / 100) + Dep * (1 - (s_FW * FW_ / 100)))\n",
" \n",
" # Evaporation\n",
" E = np.maximum((Kei + Kep) * ET0, 0)\n",
" \n",
" # Transpiration\n",
" Tr = Kcb * Ks * ET0\n",
" \n",
" # Irrigation\n",
" # model_outputs.Irr.loc[{'time': dates[0]}] = Irrig\n",
" \n",
" # Write outputs\n",
" outputs = nc.Dataset(save_path, mode='r+')\n",
" # Evaporation\n",
" outputs.variables['E'][:,:,0] = np.round(E * 1000).astype('int16')\n",
" # Transpiration\n",
" outputs.variables['Tr'][:,:,0] = np.round(Tr * 1000).astype('int16')\n",
" # Irrigation\n",
" outputs.variables['Irr'][:,:,0] = np.round(Irrig * 1000).astype('int16')\n",
" outputs.close()\n",
" \n",
" # Potential evapotranspiration and evaporative fraction ??\n",
" \n",
" # Update depletions (root and deep zones) at the end of the day\n",
" Dr = np.minimum(np.maximum(Dr + E + Tr - diff_dr, 0), TAW)\n",
" Dd = np.minimum(np.maximum(Dd + diff_dr, 0), TDW)\n",
" del E, Tr\n",
" \n",
" # Update previous day values\n",
" TAW0 = TAW\n",
" TDW0 = TDW\n",
" Dr0 = Dr\n",
" Dd0 = Dd\n",
" Zr0 = Zr\n",
" \n",
" print('day 1/', len(dates), ' ', end = '\\r')\n",
" \n",
" # # Update variable_t1 values\n",
" # for variable in calculation_variables_t1:\n",
" # variables_t1[variable] = variables_t2[variable].copy(deep = True)\n",
" \n",
" #============ Time loop ============#\n",
" for i in range(1, len(dates)):\n",
" \n",
" # Reset input aliases\n",
" # input data\n",
" ndvi = (ndvi_cube.ndvi.sel({'time': dates[i]}).to_numpy() / 255)\n",
" prec = prec_cube.prec.sel({'time': dates[i]}).to_numpy() / 1000\n",
" ET0 = ET0_cube.ET0.sel({'time': dates[i]}).to_numpy() / 1000\n",
" ET0_previous = ET0_cube.ET0.sel({'time': dates[i-1]}).to_numpy() / 1000\n",
" \n",
" # Update variables\n",
" ## Fraction cover\n",
" FCov = s_Fslope * Fslope_ * ndvi + s_Foffset * Foffset_\n",
" FCov = np.minimum(np.maximum(FCov, 0), s_Fc_stop * Fc_stop_)\n",
" \n",
" ## Root depth upate\n",
" Zr = s_minZr * minZr_ + (FCov / (s_FmaxFC * FmaxFC_)) * s_maxZr * (maxZr_ - minZr_)\n",
" \n",
" # Water capacities\n",
" TAW = (soil_params.FC.values - soil_params.WP.values) * Zr\n",
" TDW = (soil_params.FC.values - soil_params.WP.values) * (s_Zsoil * Zsoil_ - Zr)\n",
" \n",
" # Update depletions\n",
" Dr = update_Dr(TAW, TDW, Zr, TAW0, TDW0, Dr0, Dd0, Zr0)\n",
" Dd = update_Dd(TAW, TDW, Zr, TAW0, TDW0, Dd0, Zr0)\n",
" \n",
" # Update param p\n",
" p_ = (np.minimum(np.maximum(s_p * p_ + 0.04 * (5 - ET0_previous), 0.1), 0.8) * (1 / s_p)).round(0).astype('i2')\n",
" \n",
" # Irrigation ==============!!!!!\n",
" Irrig = np.minimum(np.maximum(Dr - prec, 0), s_Lame_max * Lame_max_) * Irrig_auto_\n",
" Irrig = np.where(Dr > TAW * s_p * p_, Irrig, 0)\n",
" \n",
" # Kcb\n",
" Kcb = np.minimum(s_Kslope * Kslope_ * ndvi + s_Koffset * Koffset_, s_KmaxKcb * KmaxKcb_)\n",
" \n",
" # # Write outputs\n",
" # model_outputs.Irr.loc[{'time': dates[i]}] = Irrig\n",
" \n",
" # Update depletions with rainfall and/or irrigation \n",
" \n",
" # Write outputs\n",
" # Variable directly written since not used later\n",
" outputs = nc.Dataset(save_path, mode='r+')\n",
" ## DP (Deep percolation)\n",
" outputs.variables['DP'][:,:,i] = np.round(-np.minimum(Dd + np.minimum(Dr - prec - Irrig, 0), 0) * 1000).astype('int16')\n",
" outputs.close()\n",
" \n",
" # model_outputs.DP.loc[{'time': dates[i]}] = -np.minimum(Dd + np.minimum(Dr - prec - Irrig, 0), 0)\n",
" \n",
" ## De\n",
" Dei = np.minimum(np.maximum(Dei - prec - Irrig / (s_FW * FW_ / 100), 0), TEW)\n",
" Dep = np.minimum(np.maximum(Dep - prec, 0), TEW)\n",
" \n",
" fewi = np.minimum(1 - FCov, (s_FW * FW_ / 100))\n",
" fewp = 1 - FCov - fewi\n",
" \n",
" De = (Dei * fewi + Dep * fewp) / (fewi + fewp)\n",
" De = np.where(np.isfinite(De), De, Dei * (s_FW * FW_ / 100) + Dep * (1 - (s_FW * FW_ / 100)))\n",
"\n",
" ## Dr\n",
" Dr = np.minimum(np.maximum(Dr - prec - Irrig, 0), TAW)\n",
" \n",
" ## Dd\n",
" Dd = np.minimum(np.maximum(Dd + np.minimum(Dr - prec - Irrig, 0), 0), TDW)\n",
" \n",
" # Diffusion coefficients\n",
" diff_rei = calculate_diff_re(TAW, Dr, Zr, RUE, Dei, FCov, Ze_, DiffE_, scale_factor)\n",
" diff_rep = calculate_diff_re(TAW, Dr, Zr, RUE, Dep, FCov, Ze_, DiffE_, scale_factor)\n",
" diff_dr = calculate_diff_dr(TAW, TDW, Dr, Zr, Dd, FCov, Zsoil_, DiffR_, scale_factor) \n",
" \n",
" # Weighing factor W\n",
" W = calculate_W(TEW, Dei, Dep, fewi, fewp)\n",
" \n",
" # Write outputs\n",
" # Variables directly written since not used later\n",
" outputs = nc.Dataset(save_path, mode='r+')\n",
" # Soil water content of evaporative layer\n",
" outputs.variables['SWCe'][:,:,i] = np.round((1 - De/TEW) * 1000).astype('int16')\n",
" # Soil water content of root layer\n",
" outputs.variables['SWCe'][:,:,i] = np.round((1 - Dr/TAW) * 1000).astype('int16')\n",
" outputs.close()\n",
" \n",
" \n",
" # model_outputs.SWCe.loc[{'time': dates[i]}] = 1 - De/TEW\n",
" # model_outputs.SWCr.loc[{'time': dates[i]}] = 1 - Dr/TAW\n",
" \n",
" # Water Stress coefficient\n",
" Ks = np.minimum((TAW - Dr) / (TAW * (1 - s_p * p_)), 1)\n",
" \n",
" # Reduction coefficient for evaporation\n",
" Kei = np.minimum(W * calculate_Kr(TEW, Dei, REW_, scale_factor) * (s_Kcmax * Kcmax_ - Kcb), fewi * s_Kcmax * Kcmax_)\n",
" Kep = np.minimum((1 - W) * calculate_Kr(TEW, Dei, REW_, scale_factor) * (s_Kcmax * Kcmax_ - Kcb), fewp * s_Kcmax * Kcmax_)\n",
" \n",
" # Prepare coefficients for evapotranspiration\n",
" Kti = np.minimum(((s_Ze * Ze_ / Zr)**6) * (1 - Dei / TEW) / np.maximum(1 - Dr / TAW, 0.001), 1)\n",
" Ktp = np.minimum(((s_Ze * Ze_ / Zr)**6) * (1 - Dep / TEW) / np.maximum(1 - Dr / TAW, 0.001), 1)\n",
" Tei = Kti * Ks * Kcb * ET0\n",
" Tep = Ktp * Ks * Kcb * ET0\n",
" \n",
" # Update depletions\n",
" Dei = np.where(fewi > 0, np.minimum(np.maximum(Dei + ET0 * Kei / fewi + Tei - diff_rei, 0), TEW), np.minimum(np.maximum(Dei + Tei - diff_rei, 0), TEW))\n",
" Dep = np.where(fewp > 0, np.minimum(np.maximum(Dep + ET0 * Kep / fewp + Tep - diff_rep, 0), TEW), np.minimum(np.maximum(Dep + Tep - diff_rep, 0), TEW))\n",
" \n",
" De = (Dei * fewi + Dep * fewp) / (fewi + fewp)\n",
" De = np.where(np.isfinite(De), De, Dei * (s_FW * FW_ / 100) + Dep * (1 - (s_FW * FW_ / 100)))\n",
" \n",
" # Evaporation\n",
" E = np.maximum((Kei + Kep) * ET0, 0)\n",
" \n",
" # Transpiration\n",
" Tr = Kcb * Ks * ET0\n",
" \n",
" # Write outputs\n",
" outputs = nc.Dataset(save_path, mode='r+')\n",
" # Evaporation\n",
" outputs.variables['E'][:,:,i] = np.round(E * 1000).astype('int16')\n",
" # Transpiration\n",
" outputs.variables['Tr'][:,:,i] = np.round(Tr * 1000).astype('int16')\n",
" # Irrigation\n",
" outputs.variables['Irr'][:,:,i] = np.round(Irrig * 1000).astype('int16')\n",
" outputs.close()\n",
" \n",
" # Potential evapotranspiration and evaporative fraction ??\n",
" \n",
" # Update depletions (root and deep zones) at the end of the day\n",
" Dr = np.minimum(np.maximum(Dr + E + Tr - diff_dr, 0), TAW)\n",
" Dd = np.minimum(np.maximum(Dd + diff_dr, 0), TDW)\n",
" del E, Tr\n",
" \n",
" # Update previous day values\n",
" TAW0 = TAW\n",
" TDW0 = TDW\n",
" Dr0 = Dr\n",
" Dd0 = Dd\n",
" Zr0 = Zr\n",
" \n",
" # # Update variable_t1 values\n",
" # for variable in calculation_variables_t1:\n",
" # variables_t1[variable] = variables_t2[variable].copy(deep = True)\n",
" \n",
" print('day ', i+1, '/', len(dates), ' ', end = '\\r')\n",
" \n",
" # Scale the model_outputs variable to save in int16 format\n",
" # model_outputs = model_outputs * 1000\n",
" \n",
" # Save model outputs to netcdf\n",
" # model_outputs.to_netcdf(save_path, encoding = encoding_dict)\n",
" \n",
" return None"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-99049348-2ac2-11ee-856b-00155de7557f</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
"\n",
" <tr>\n",
" \n",
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> distributed.LocalCluster</td>\n",
" \n",
" </tr>\n",
"\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" \n",
"\n",
" </table>\n",
"\n",
" \n",
"\n",
" \n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
" <div class=\"jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\">\n",
" </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px; margin-top: 0px;\">LocalCluster</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">48bf00f8</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Workers:</strong> 4\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads:</strong> 8\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total memory:</strong> 23.47 GiB\n",
" </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\"><strong>Status:</strong> running</td>\n",
" <td style=\"text-align: left;\"><strong>Using processes:</strong> True</td>\n",
"</tr>\n",
"\n",
" \n",
" </table>\n",
"\n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\">\n",
" <h3 style=\"display: inline;\">Scheduler Info</h3>\n",
" </summary>\n",
"\n",
" <div style=\"\">\n",
" <div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Scheduler</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Scheduler-d5ec5b54-f595-40fe-a4fe-ab30a54fb158</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm:</strong> tcp://127.0.0.1:32951\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Workers:</strong> 4\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads:</strong> 8\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Started:</strong> Just now\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total memory:</strong> 23.47 GiB\n",
" </td>\n",
" </tr>\n",
" </table>\n",
" </div>\n",
" </div>\n",
"\n",
" <details style=\"margin-left: 48px;\">\n",
" <summary style=\"margin-bottom: 20px;\">\n",
" <h3 style=\"display: inline;\">Workers</h3>\n",
" </summary>\n",
"\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 0</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:45105\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 2\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:42443/status\" target=\"_blank\">http://127.0.0.1:42443/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 5.87 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:37411\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-v5s639lg\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 1</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:36993\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 2\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:40175/status\" target=\"_blank\">http://127.0.0.1:40175/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 5.87 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:45519\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-0cke1ycu\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 2</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:36987\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 2\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:45413/status\" target=\"_blank\">http://127.0.0.1:45413/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 5.87 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:42671\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-2gvmrt11\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 3</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:40491\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 2\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:33959/status\" target=\"_blank\">http://127.0.0.1:33959/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 5.87 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:37215\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-iv3e92vj\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
"\n",
" </details>\n",
"</div>\n",
"\n",
" </details>\n",
" </div>\n",
"</div>\n",
" </details>\n",
" \n",
"\n",
" </div>\n",
"</div>"
],
"text/plain": [
"<Client: 'tcp://127.0.0.1:32951' processes=4 threads=8, memory=23.47 GiB>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client = Client(interface='lo')\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/auclairj/anaconda3/envs/modspa_pixel/lib/python3.10/site-packages/dask/array/chunk.py:278: RuntimeWarning: invalid value encountered in cast\n",
" return x.astype(astype_dtype, **kwargs)\n",
"/home/auclairj/anaconda3/envs/modspa_pixel/lib/python3.10/site-packages/dask/array/chunk.py:278: RuntimeWarning: invalid value encountered in cast\n",
" return x.astype(astype_dtype, **kwargs)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"day 1/ 366 \r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/auclairj/anaconda3/envs/modspa_pixel/lib/python3.10/site-packages/dask/array/chunk.py:278: RuntimeWarning: invalid value encountered in cast\n",
" return x.astype(astype_dtype, **kwargs)\n",
"/home/auclairj/anaconda3/envs/modspa_pixel/lib/python3.10/site-packages/dask/array/chunk.py:278: RuntimeWarning: invalid value encountered in cast\n",
" return x.astype(astype_dtype, **kwargs)\n",
"/tmp/ipykernel_1387/2938761162.py:242: RuntimeWarning: divide by zero encountered in divide\n",
" tmp1 = (((TAW - Dr) / Zr - (RUE - De) / (scale_dict['Ze'] * Ze_)) / FCov) * (scale_dict['DiffE'] * DiffE_)\n",
"/tmp/ipykernel_1387/2938761162.py:285: RuntimeWarning: divide by zero encountered in divide\n",
" tmp1 = (((TDW - Dd) / (scale_dict['Zsoil'] * Zsoil_ - Zr) - (TAW - Dr) / Zr) / FCov) * scale_dict['DiffR'] * DiffR_\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"day 3 / 366 \r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/auclairj/anaconda3/envs/modspa_pixel/lib/python3.10/site-packages/dask/array/chunk.py:278: RuntimeWarning: invalid value encountered in cast\n",
" return x.astype(astype_dtype, **kwargs)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"day 366 / 366 \r"
]
}
],
"source": [
"data_path = '/mnt/e/DATA/DEV_inputs_test'\n",
"\n",
"size = 100\n",
"\n",
"ndvi_path = data_path + os.sep + 'ndvi_' + str(size) + '.nc'\n",
"prec_path = data_path + os.sep + 'rain_' + str(size) + '.tif'\n",
"ET0_path = data_path + os.sep + 'ET0_' + str(size) + '.tif'\n",
"land_cover_path = data_path + os.sep + 'land_cover_' + str(size) + '.nc'\n",
"json_config_file = '/home/auclairj/GIT/modspa-pixel/config/config_modspa.json'\n",
"param_file = '/home/auclairj/GIT/modspa-pixel/parameters/csv_files/params_samir_test.csv'\n",
"soil_path = data_path + os.sep + 'soil_' + str(size) + '.nc'\n",
"save_path = data_path + os.sep + 'outputs_' + str(size) + '.nc'\n",
"\n",
"chunk_size = {'x': 50, 'y': 50, 'time': -1}\n",
"\n",
"t = time()\n",
"\n",
"\n",
"run_samir(json_config_file, param_file, ndvi_path, prec_path, ET0_path, soil_path, land_cover_path, chunk_size, save_path)\n",
"\n",
"format_duration(time() - t)\n",
"\n",
"client.close()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/mnt/e/DATA/DEV_inputs_test/outputs_10.nc\n"
]
}
],
"source": [
"print(save_path)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "modspa_pixel",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}