diff --git a/ascii.py b/ascii.py index df700d3160c08dcfeee629e62b19b9820af7aac4..446c398b8f691b1522e38ee9cd7be0fe03972736 100644 --- a/ascii.py +++ b/ascii.py @@ -7,7 +7,7 @@ import tools import numpy as np from datetime import datetime -def writeHeaderProfile(hdrFile, cfg, device, fe, r): +def writeHeaderProfile(hdrFile, cfg, device, fe): f = open(hdrFile, 'w') # first line, header ex: # PIRATA-FR30 THALASSA IRD SBE911+ 09P-1263 BOURLES @@ -76,7 +76,7 @@ def writeHeaderProfile(hdrFile, cfg, device, fe, r): f.close() -def writeDataProfile(dataFile, cfg, device, fe, r): +def writeDataProfile(dataFile, cfg, device, fe): f = open(dataFile, 'w') # write header, first line @@ -130,23 +130,23 @@ def writeDataProfile(dataFile, cfg, device, fe, r): f.write("\n") f.close() -def writeProfile(cfg, device, fe, r): +def writeProfile(cfg, device, fe): if not os.path.exists(cfg['global']['ASCII']): os.makedirs(cfg['global']['ASCII']) fileName = "{}/{}.{}".format(cfg['global']['ASCII'], cfg['cruise']['CYCLEMESURE'], device.lower()) - writeHeaderProfile(fileName, cfg, device.lower(), fe, r, ) + writeHeaderProfile(fileName, cfg, device.lower(), fe) print('writing header file: {}'.format(fileName), end='', flush=True) print(' done...') fileName = "{}/{}_{}".format(cfg['global']['ASCII'], cfg['cruise']['CYCLEMESURE'], device.lower()) print('writing data file: {}'.format(fileName), end='', flush=True) - writeDataProfile(fileName, cfg, device.lower(), fe, r) + writeDataProfile(fileName, cfg, device.lower(), fe) print(' done...') -def writeDecimalDataTrajectory(dataFile, cfg, device, fe, r): +def writeDecimalDataTrajectory(dataFile, cfg, device, fe): f = open(dataFile, 'w') # write header, first line @@ -175,7 +175,7 @@ def writeDecimalDataTrajectory(dataFile, cfg, device, fe, r): f.write("\n") f.close() -def writeHumanDataTrajectory(dataFile, cfg, device, fe, r): +def writeHumanDataTrajectory(dataFile, cfg, device, fe): f = open(dataFile, 'w') # write header, first line @@ -219,7 +219,7 @@ def writeHumanDataTrajectory(dataFile, cfg, device, fe, r): f.close() -def writeTrajectory(cfg, device, fe, r): +def writeTrajectory(cfg, device, fe): if not os.path.exists(cfg['global']['ASCII']): os.makedirs(cfg['global']['ASCII']) @@ -227,13 +227,13 @@ def writeTrajectory(cfg, device, fe, r): fileName = "{}/{}.{}".format(cfg['global']['ASCII'], cfg['cruise']['CYCLEMESURE'], device.lower()) print('writing file: {}'.format(fileName), end='', flush=True) - writeHumanDataTrajectory(fileName, cfg, device.lower(), fe, r, ) + writeHumanDataTrajectory(fileName, cfg, device.lower(), fe) print(' done...') # write ascii file with decimal time and position fileName = "{}/{}_{}".format(cfg['global']['ASCII'], cfg['cruise']['CYCLEMESURE'], device.lower()) print('writing file: {}'.format(fileName), end='', flush=True) - writeDecimalDataTrajectory(fileName, cfg, device.lower(), fe, r) + writeDecimalDataTrajectory(fileName, cfg, device.lower(), fe) print(' done...') \ No newline at end of file diff --git a/config.toml b/config.toml index 4258a6e8ab1a9acc3a6e7f56b6cabb5fb6a47972..a751db1468af8bb2dfa22438f780f1d8a013ad32 100644 --- a/config.toml +++ b/config.toml @@ -279,7 +279,7 @@ julianOrigin = 1 [tsg.header] isHeader = '^[*#]+\s+' - endHeader = '^[*]END[*]' + endHeader = '^[*]END[*]$' DATETIME = 'System UpLoad Time\s*=\s*(\w+)\s+(\d+)\s+(\d+)\s+(\d+):(\d+):(\d+)' [tsg.split] diff --git a/netcdf.py b/netcdf.py index 76b11d8b83c4bd4ea7cbb48aae97940919167098..0bba2507859dfcfc060d50c19c33b7bd14efa359 100644 --- a/netcdf.py +++ b/netcdf.py @@ -37,7 +37,7 @@ def writeGlobalAttributes(nc, cfg, device, type): nc.Netcdf_version = "3.6" -def writeProfile(cfg, device, fe, r): +def writeProfile(cfg, device, fe): # ncvars is a dictionary that store a NETCDF variable for each physical parameter key ncvars = {} @@ -81,7 +81,7 @@ def writeProfile(cfg, device, fe, r): print(f"Define variable : {key}") # for each variables get the attributes dictionary from Roscop - hash = r[key] + hash = fe.roscop[key] # _FillValue attribute must be set when variable is created # (using fill_value keyword to createVariable) if '_FillValue' in hash: @@ -131,7 +131,7 @@ def writeProfile(cfg, device, fe, r): nc.close() print('done...') -def writeTrajectory(cfg, device, fe, r): +def writeTrajectory(cfg, device, fe): # ncvars is a dictionary that store a NETCDF variable for each physical parameter key ncvars = {} @@ -171,7 +171,7 @@ def writeTrajectory(cfg, device, fe, r): print(f"Define variable : {key}") # for each variables get the attributes dictionary from Roscop - hash = r[key] + hash = fe.roscop[key] # _FillValue attribute must be set when variable is created # (using fill_value keyword to createVariable) if '_FillValue' in hash: diff --git a/odv.py b/odv.py index 204e02e4177da00431f178a66d557b834d5d8c54..d22ec8e238368556aa0d6b55468d4257292732c7 100644 --- a/odv.py +++ b/odv.py @@ -7,105 +7,103 @@ from re import S import tools import numpy as np from datetime import datetime - -def writeProfile(cfg, device, fe, r): + +def writeHeader(cfg, device, dataType): + '''Specifying * for Type (dataType) lets ODV make the choice. + If Bot. Depth values are not available, you should leave this field empty.''' if not os.path.exists(cfg['global']['odv']): os.makedirs(cfg['global']['odv']) fileName = "{}/{}_{}_odv.txt".format(cfg['global']['odv'], cfg['cruise']['CYCLEMESURE'], device.lower()) print('writing data file: {}'.format(fileName), end='', flush=True) - f = open(fileName, 'w') + fd = open(fileName, 'w') # write odv header - f.write(f"//ODV Spreadsheet file : {fileName}\n") - f.write(f"//Data treated : {datetime.today().strftime('%Y-%m-%dT%H:%M:%SZ')}\n") - f.write("//<DataType>Profiles</DataType>\n") - f.write(f"//<InstrumentType>{cfg[device.lower()]['typeInstrument']}</InstrumentType>\n") - f.write(f"//<Source>{os.getcwd()}</Sources>\n") - f.write(f"//<Creator>{cfg['cruise']['CREATOR']}</Creator>\n") - f.write("//\n") - f.write("Cruise\tStation\tType\tyyyy-mm-ddThh:mm:ss.sss\tLongitude [degrees_east]\tLatitude [degrees_north]\tBot. Depth [m]") + fd.write(f"//ODV Spreadsheet file : {fileName}\n") + fd.write(f"//Data treated : {datetime.today().strftime('%Y-%m-%dT%H:%M:%SZ')}\n") + fd.write(f"//<DataType>{dataType}</DataType>\n") + fd.write(f"//<InstrumentType>{cfg[device.lower()]['typeInstrument']}</InstrumentType>\n") + fd.write(f"//<Source>{os.getcwd()}</Sources>\n") + fd.write(f"//<Creator>{cfg['cruise']['CREATOR']}</Creator>\n") + fd.write("//\n") + fd.write(f"Cruise\tStation\tType\tyyyy-mm-ddThh:mm:ss.sss\tLongitude [degrees_east]\tLatitude [degrees_north]\tBot. Depth [m]") + return fd + +def writeProfile(cfg, device, fe): + + # You should use B for stations with less than about 250 samples (e.g., bottle data) and C for stations + # with more than about 250 samples (e.g., CTD, XBT, etc.) + if cfg[device.lower()] == 'btl': + stationType = 'B' + else: + stationType = 'C' - # write variables and unit + # write header and return file descriptor + fd = writeHeader(cfg, device, 'Profiles') + # write following variables and unit for k in fe.keys: - f.write(f"\t{k} [{fe.roscop[k]['units']}]") - f.write("\n") + fd.write(f"\t{k} [{fe.roscop[k]['units']}]") + fd.write("\n") - if cfg[device.lower()] == 'btl': - type = 'B' - else: - type = 'C' for i in range(fe.n): # for each valid line in array (not _fillvalue) for m in range(fe.size[i+1]): fmt = fe.roscop['PROFILE']['format'] - f.write(f"%s\t{fmt}\t%s" % (cfg['cruise']['CYCLEMESURE'], fe['PROFILE'][i], type)) - f.write(f"\t%s" % (tools.julian2format(fe['TIME'][i], "%Y-%m-%dT%H:%M:%S"))) + fd.write(f"%s\t{fmt}\t%s" % (cfg['cruise']['CYCLEMESURE'], fe['PROFILE'][i], stationType)) + fd.write(f"\t%s" % (tools.julian2format(fe['TIME'][i], "%Y-%m-%dT%H:%M:%S"))) fmt = fe.roscop['LONGITUDE']['format'] - f.write(f"\t{fmt}" % (fe['LONGITUDE'][i])) + fd.write(f"\t{fmt}" % (fe['LONGITUDE'][i])) fmt = fe.roscop['LATITUDE']['format'] - f.write(f"\t{fmt}" % (fe['LATITUDE'][i])) - if fe.iskey('BATH'): + fd.write(f"\t{fmt}" % (fe['LATITUDE'][i])) + if fe.iskey('BATH') and not np.isnan(fe['BATH'][i]): fmt = fe.roscop['BATH']['format'] - f.write(f"\t{fmt}" % (fe['BATH'][i])) + fd.write(f"\t{fmt}" % (fe['BATH'][i])) else: - f.write("\t") - for k in fe.keys: - if fe[k][i][m] != fe.roscop[k]['_FillValue']: + fd.write("\t") + for k in fe.keys: + if not np.isnan(fe[k][i][m]): + #if fe[k][i][m] != fe.roscop[k]['_FillValue']: fmt = fe.roscop[k]['format'] - f.write(f"\t{fmt}" % (fe[k][i][m])) + fd.write(f"\t{fmt}" % (fe[k][i][m])) else: - f.write("\t") - f.write("\n") - f.close() + fd.write("\t") + fd.write("\n") + fd.close() print(' done...') -def writeTrajectory(cfg, device, fe, r): - if not os.path.exists(cfg['global']['odv']): - os.makedirs(cfg['global']['odv']) +def writeTrajectory(cfg, device, fe): - fileName = "{}/{}_{}_odv.txt".format(cfg['global']['odv'], - cfg['cruise']['CYCLEMESURE'], device.lower()) - print('writing data file: {}'.format(fileName), end='', flush=True) - f = open(fileName, 'w') - - # write odv header - f.write(f"//ODV Spreadsheet file : {fileName}\n") - f.write(f"//Data treated : {datetime.today().strftime('%Y-%m-%dT%H:%M:%SZ')}\n") - f.write("//<DataType>Profiles</DataType>\n") - f.write(f"//<InstrumentType>{cfg[device.lower()]['typeInstrument']}</InstrumentType>\n") - f.write(f"//<Source>{os.getcwd()}</Sources>\n") - f.write(f"//<Creator>{cfg['cruise']['CREATOR']}</Creator>\n") - f.write("//\n") - f.write("Cruise\tStation\tType\tyyyy-mm-ddThh:mm:ss.sss\tLongitude [degrees_east]\tLatitude [degrees_north]\tBot. Depth [m]") - + # write header and return file descriptor + fd = writeHeader(cfg, device, 'Trajectories') # write variables and unit for k in fe.keys: if k == 'ID' or k == 'DAYD' or k == 'LATITUDE' or k == 'LONGITUDE': continue else: - f.write(f"\t{k} [{fe.roscop[k]['units']}]") - f.write("\n") + fd.write(f"\t{k} [{fe.roscop[k]['units']}]") + fd.write("\n") - type = 'B' + # If Bot. Depth values are not available, you should leave this field empty, + # usually the case for trajectory + stationType = '' for i in range(fe.n): # Trajectory data of this form have exactly one sample per station. The number of stations # equals the number of observation along the trajectory, which is usually large. fmt = "%06d" - f.write(f"%s\t{fmt}\t%s" % (cfg['cruise']['CYCLEMESURE'], i+1, type)) - f.write(f"\t%s" % (tools.julian2format(fe['DAYD'][i], "%Y-%m-%dT%H:%M:%S"))) + fd.write(f"%s\t{fmt}\t%s" % (cfg['cruise']['CYCLEMESURE'], i+1, stationType)) + fd.write(f"\t%s" % (tools.julian2format(fe['DAYD'][i], "%Y-%m-%dT%H:%M:%S"))) fmt = fe.roscop['LONGITUDE']['format'] - f.write(f"\t{fmt}" % (fe['LONGITUDE'][i])) + fd.write(f"\t{fmt}" % (fe['LONGITUDE'][i])) fmt = fe.roscop['LATITUDE']['format'] - f.write(f"\t{fmt}" % (fe['LATITUDE'][i])) - if fe.iskey('BATH'): + fd.write(f"\t{fmt}" % (fe['LATITUDE'][i])) + if fe.iskey('BATH') and not np.isnan(fe['BATH'][i]): fmt = fe.roscop['BATH']['format'] - f.write(f"\t{fmt}" % (fe['BATH'][i])) + fd.write(f"\t{fmt}" % (fe['BATH'][i])) else: - f.write("\t") + fd.write("\t") for k in fe.keys: if k == 'ID' or k == 'DAYD' or k == 'LATITUDE' or k == 'LONGITUDE': @@ -113,11 +111,11 @@ def writeTrajectory(cfg, device, fe, r): else: if fe[k][i] != fe.roscop[k]['_FillValue']: fmt = fe.roscop[k]['format'] - f.write(f"\t{fmt}" % (fe[k][i])) + fd.write(f"\t{fmt}" % (fe[k][i])) else: - f.write("\t") - f.write("\n") - f.close() + fd.write("\t") + fd.write("\n") + fd.close() print(' done...') diff --git a/profile.py b/profile.py index 85a8069f53c01b35a5a3ca8e4b1fd270907104ac..0734402d673199de1499377e3a97400a928d32b8 100644 --- a/profile.py +++ b/profile.py @@ -519,14 +519,14 @@ class Profile: self.read_files(cfg, ti) # write ASCII hdr and data files - ascii.writeProfile(cfg, ti, self, self.roscop) + ascii.writeProfile(cfg, ti, self) # write ODV file if cfg['global']['odv']: - odv.writeProfile(cfg, ti, self, self.roscop) + odv.writeProfile(cfg, ti, self) # write the NetCDF file - netcdf.writeProfile(cfg, ti, self, self.roscop) + netcdf.writeProfile(cfg, ti, self) # for testing in standalone context