Skip to content
Snippets Groups Projects
Commit 9d288c59 authored by jacques.grelet_ird.fr's avatar jacques.grelet_ird.fr
Browse files

remove CTD default key in cmd line 

move Roscop read from netcdf to oceano
parent 795a6100
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
code_roscop.csv
+ 11
− 10
View file @ 9d288c59
...@@ -5,7 +5,7 @@ PROFILE;int32;profile number;;1;;;;;;%4d;99999;;;Station number, a station could ...@@ -5,7 +5,7 @@ PROFILE;int32;profile number;;1;;;;;;%4d;99999;;;Station number, a station could
LATITUDE;float32;latitude of measurement;latitude;degrees_north;;;;;;;;Y;;;;;;WGS84;urn:ogc:crs:EPSG::4326 LATITUDE;float32;latitude of measurement;latitude;degrees_north;;;;;;;;Y;;;;;;WGS84;urn:ogc:crs:EPSG::4326
LONGITUDE;float32;longitude of measurement;longitude;degrees_east;;;;;;;;X;;;;;;WGS84;urn:ogc:crs:EPSG::4327 LONGITUDE;float32;longitude of measurement;longitude;degrees_east;;;;;;;;X;;;;;;WGS84;urn:ogc:crs:EPSG::4327
TIME;float64;time of measurement;time;days since 1950-01-01T00:00:00Z;Relative julian days with decimal part (as parts of the day);;;0.0;90000.0;%6.6f;;T;;;;;;; TIME;float64;time of measurement;time;days since 1950-01-01T00:00:00Z;Relative julian days with decimal part (as parts of the day);;;0.0;90000.0;%6.6f;;T;;;;;;;
ETDD;float32;ellapsed time;;1;;;;0;90000;%9.5f;;;;;;;;; ETDD;float64;ellapsed time;;1;;;;0;90000;%9.5f;1E+36;;;;;;;;
DEPH;float32;depth of measurement;depth;meter;;;down;0;9000;%7.2f;1E+36;Z;;;;;;;urn:ogc:crs:EPSG::5831 DEPH;float32;depth of measurement;depth;meter;;;down;0;9000;%7.2f;1E+36;Z;;;;;;;urn:ogc:crs:EPSG::5831
DEPTH;float32;depth of measurement;depth;meter;;;down;0;9000;%7.2f;1E+36;Z;;;;;;;urn:ogc:crs:EPSG::5831 DEPTH;float32;depth of measurement;depth;meter;;;down;0;9000;%7.2f;1E+36;Z;;;;;;;urn:ogc:crs:EPSG::5831
PRES;float32;sea water pressure;sea_water_pressure;decibar;;TIME DEPTH LATITUDE LONGITUDE;;0;9000;%7.2f;1E+36;;;Pressure (dbar) of the measurement;;;;; PRES;float32;sea water pressure;sea_water_pressure;decibar;;TIME DEPTH LATITUDE LONGITUDE;;0;9000;%7.2f;1E+36;;;Pressure (dbar) of the measurement;;;;;
...@@ -21,13 +21,14 @@ WMSP;float32;wind speed - max aver per 2 mn;wind_speed;meter/second;;TIME DEPTH ...@@ -21,13 +21,14 @@ WMSP;float32;wind speed - max aver per 2 mn;wind_speed;meter/second;;TIME DEPTH
WDIR;float32;wind direction rel. true north;wind_to_direction;degree;;TIME DEPTH LATITUDE LONGITUDE;;0;360;%5.1f;1E+36;;0.001;Horizontal wind speed;;;;; WDIR;float32;wind direction rel. true north;wind_to_direction;degree;;TIME DEPTH LATITUDE LONGITUDE;;0;360;%5.1f;1E+36;;0.001;Horizontal wind speed;;;;;
ATMS;float32;atmospheric pressure - sea lev;air_pressure_at_sea_level;hectoPascal;;TIME DEPTH LATITUDE LONGITUDE;;940;1030;%8.3f;1E+36;;0.001;Atmospheric pressure at sea level;;;;; ATMS;float32;atmospheric pressure - sea lev;air_pressure_at_sea_level;hectoPascal;;TIME DEPTH LATITUDE LONGITUDE;;940;1030;%8.3f;1E+36;;0.001;Atmospheric pressure at sea level;;;;;
RELH;float32;relative humidity;relative_humidity;%;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%6.1f;1E+36;;0.001;Relative humidity;;;;; RELH;float32;relative humidity;relative_humidity;%;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%6.1f;1E+36;;0.001;Relative humidity;;;;;
NSCT;float32;current north component;;cm/second;;TIME DEPTH LATITUDE LONGITUDE;;-100;100;%7.3f;1E+36;;0.01; Current north;;;;; NSCT;float32;current north component;;cm/second;;TIME DEPTH LATITUDE LONGITUDE;;-100;100;%7.3f;1E+36;;0.01;" Current north";;;;;
EWCT;float32;current east component;;cm/second;;TIME DEPTH LATITUDE LONGITUDE;;-100;100;%7.3f;1E+36;;0.02;Current east;;;;; EWCT;float32;current east component;;cm/second;;TIME DEPTH LATITUDE LONGITUDE;;-100;100;%7.3f;1E+36;;0.02;Current east;;;;;
SSJT;float32;sea surface water jacket temperature;sea_water_temperature;degree_Celsius;;TIME DEPTH LATITUDE LONGITUDE;;-1.5;38;%6.3f;1E+36;;0.001;Ocean temperature;;;;; SSJT;float32;sea surface water jacket temperature;sea_water_temperature;degree_Celsius;;TIME DEPTH LATITUDE LONGITUDE;;-1.5;38;%6.3f;1E+36;;0.001;Ocean temperature;;;;;
SSTP;float32;sea surface water temperature;sea_water_temperature;degree_Celsius;;TIME DEPTH LATITUDE LONGITUDE;;-1.5;38;%6.3f;1E+36;;0.001;Sea surface Ocean temperature;;;;; SSTP;float32;sea surface water temperature;sea_water_temperature;degree_Celsius;;TIME DEPTH LATITUDE LONGITUDE;;-1.5;38;%6.3f;1E+36;;0.001;Sea surface Ocean temperature;;;;;
SSPS;float32;sea surface practical salinity;sea_water_salinity ;pss;;TIME DEPTH LATITUDE LONGITUDE;;-1.5;38;%6.3f;1E+36;;0.001;Ocean salinity;;;;; SSPS;float32;sea surface practical salinity;"sea_water_salinity ";pss;;TIME DEPTH LATITUDE LONGITUDE;;-1.5;38;%6.3f;1E+36;;0.001;Ocean salinity;;;;;
PCO2;float32;co2 part. pres in dry/wet gas;;microatmosphere;;TIME DEPTH LATITUDE LONGITUDE;;100;700;%7.3f;1E+36;;0.001;;;;;; PCO2;float32;co2 part. pres in dry/wet gas;;microatmosphere;;TIME DEPTH LATITUDE LONGITUDE;;100;700;%7.3f;1E+36;;0.001;;;;;;
FLU2;float32;fluorescence;;milligram/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;1;%8.4f;1E+36;;0.001;;;;;; FLU2;float32;fluorescence;;milligram/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;1;%8.4f;1E+36;;0.001;;;;;;
FLU3;float32;secondary fluorescence;;milligram/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;1;%8.4f;1E+36;;0.001;;;;;;
TUR1;float32;light diffusion coefficient;;m-1;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;; TUR1;float32;light diffusion coefficient;;m-1;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;;
TUR2;float32;light attenuation coefficient;;m-1;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;; TUR2;float32;light attenuation coefficient;;m-1;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;;
TUR3;float32;light transmission;;%;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%6.2f;1E+36;;0.001;;;;;; TUR3;float32;light transmission;;%;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%6.2f;1E+36;;0.001;;;;;;
...@@ -54,10 +55,10 @@ DOV1;float32;dissolved oxygen primary sensor voltage;;V;;TIME DEPTH LATITUDE LON ...@@ -54,10 +55,10 @@ DOV1;float32;dissolved oxygen primary sensor voltage;;V;;TIME DEPTH LATITUDE LON
DOV2;float32;dissolved oxygen secondary sensor voltage;;V;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.4f;1E+36;;0.001;;;;;; DOV2;float32;dissolved oxygen secondary sensor voltage;;V;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.4f;1E+36;;0.001;;;;;;
DVT1;float32;dissolved oxygen primary sensor dv/dt;;dv/dt;;TIME DEPTH LATITUDE LONGITUDE;;-1;1;%+7.5f;1E+36;;0.001;;;;;; DVT1;float32;dissolved oxygen primary sensor dv/dt;;dv/dt;;TIME DEPTH LATITUDE LONGITUDE;;-1;1;%+7.5f;1E+36;;0.001;;;;;;
DVT2;float32;dissolved oxygen secondary sensor dv/dt;;dv/dt;;TIME DEPTH LATITUDE LONGITUDE;;-1;1;%+7.5f;1E+36;;0.002;;;;;; DVT2;float32;dissolved oxygen secondary sensor dv/dt;;dv/dt;;TIME DEPTH LATITUDE LONGITUDE;;-1;1;%+7.5f;1E+36;;0.002;;;;;;
NTIW;float32;nitrite (no2-n) ;moles_of_nitrite_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%7.3f;1E+36;;0.001;;;;;; NTIW;float32;"nitrite (no2-n) ";moles_of_nitrite_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%7.3f;1E+36;;0.001;;;;;;
NTRI;float32;nitrite (no2-n) ;moles_concentration_of_nitrite_in_sea_water;millimole/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;; NTRI;float32;"nitrite (no2-n) ";moles_concentration_of_nitrite_in_sea_water;millimole/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;;
NTAW;float32;nitrate (no3-n) ;moles_of_nitrate_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;90;%5.2f;1E+36;;0.001;;;;;; NTAW;float32;"nitrate (no3-n) ";moles_of_nitrate_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;90;%5.2f;1E+36;;0.001;;;;;;
NTRA;float32;nitrate (no3-n) ;moles_concentration_of_nitrate_in_sea_water;millimole/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;56;%6.3f;1E+36;;0.001;;;;;; NTRA;float32;"nitrate (no3-n) ";moles_concentration_of_nitrate_in_sea_water;millimole/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;56;%6.3f;1E+36;;0.001;;;;;;
NTZW;float32;nitrate + nitrite;moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;; NTZW;float32;nitrate + nitrite;moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%6.3f;1E+36;;0.001;;;;;;
NTRZ;float32;nitrate + nitrite;moles_concentration_of_nitrate_and_nitrite_in_sea_water;millimole/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%6.2f;1E+36;;0.001;;;;;; NTRZ;float32;nitrate + nitrite;moles_concentration_of_nitrate_and_nitrite_in_sea_water;millimole/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;100;%6.2f;1E+36;;0.001;;;;;;
PHOW;float32;phosphate (po4-p);moles_of_phosphate_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%5.3f;1E+36;;0.001;;;;;; PHOW;float32;phosphate (po4-p);moles_of_phosphate_per_unit_mass_in_sea_water;micromole/kg;;TIME DEPTH LATITUDE LONGITUDE;;0;10;%5.3f;1E+36;;0.001;;;;;;
...@@ -65,7 +66,7 @@ PHOS;float32;phosphate (po4-p);moles_phosphate_of_nitrate_in_sea_water;millimole ...@@ -65,7 +66,7 @@ PHOS;float32;phosphate (po4-p);moles_phosphate_of_nitrate_in_sea_water;millimole
PHPT;float32;total pheophytin;;milligramm/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;99;%5.2f;1E+36;;0.01;;;;;; PHPT;float32;total pheophytin;;milligramm/m3;;TIME DEPTH LATITUDE LONGITUDE;;0;99;%5.2f;1E+36;;0.01;;;;;;
WETT;float32;wet bulb temperature;;degree_Celsius;;TIME DEPTH LATITUDE LONGITUDE;;0;90;%5.1f;1E+36;;0.001;;;;;; WETT;float32;wet bulb temperature;;degree_Celsius;;TIME DEPTH LATITUDE LONGITUDE;;0;90;%5.1f;1E+36;;0.001;;;;;;
HEIG;float32;atmospheric height;;meter;;TIME DEPTH LATITUDE LONGITUDE;up;0;40000;%8.2f;1E+36;;0.1;;;;;;urn:ogc:crs:EPSG::5829 HEIG;float32;atmospheric height;;meter;;TIME DEPTH LATITUDE LONGITUDE;up;0;40000;%8.2f;1E+36;;0.1;;;;;;urn:ogc:crs:EPSG::5829
NAVG;int32;averaged data cycle number;;1;;;;0;999;%3.0f;99999;;;;;;;; NAVG;int32;averaged data cycle number;;1;;;;0;999;%3d;9999;;;;;;;;
BOTL;int32;sea water sample bottle number;;1;;;;1;36;%3d;99999;;;;;;;; BOTL;int32;sea water sample bottle number;;1;;;;1;36;%3d;99999;;;;;;;;
TYPECAST;int32;type of cast: unknow=0, phy=1, bio=2, geo=3;;1;;;;0.0;9.0;%1d;99999;;;;;;;; TYPECAST;int32;type of cast: unknow=0, phy=1, bio=2, geo=3;;1;;;;0;9;%1d;99999;;;;;;;;
BATH;float32;bathymetric depth;;meter;;;;0;11000;%6.1f;1E+36;;0.1;;;;;; BATH;float32;bathymetric depth;;meter;;;;0;11000;%6.1f;1E+36;;0.1;;;;;;
\ No newline at end of file
file_extractor.py
+ 96
− 79
View file @ 9d288c59
...@@ -29,11 +29,12 @@ class FileExtractor: ...@@ -29,11 +29,12 @@ class FileExtractor:
''' '''
# constructor with values by defaul # constructor with values by defaul
def __init__(self, fname, keys, separator=None): def __init__(self, fname, roscop, keys, separator=None):
# attibutes # attibutes
# public: # public:
self.fname = fname self.fname = fname
self.keys = keys self.keys = keys
self.roscop = roscop
self.n = 0 self.n = 0
self.m = 0 self.m = 0
self.lineHeader = 0 self.lineHeader = 0
...@@ -44,7 +45,7 @@ class FileExtractor: ...@@ -44,7 +45,7 @@ class FileExtractor:
self.__data = {} self.__data = {}
self.__regex = {} self.__regex = {}
# replace this constante with roscop fill value # replace this constante with roscop fill value
self.__FillValue = 1e36 #self.__FillValue = 1e36
# overloading operators # overloading operators
def __getitem__(self, key): def __getitem__(self, key):
...@@ -82,11 +83,9 @@ class FileExtractor: ...@@ -82,11 +83,9 @@ class FileExtractor:
# first pass on file(s) # first pass on file(s)
d = cfg[ti.lower()]['header'] d = cfg[ti.lower()]['header']
#print(d, end='\n') # fill the __regex dict with compiled regex
for key in d.keys(): for key in d.keys():
print("{}: {}".format(key, d[key]))
self.__regex[key] = re.compile(d[key]) self.__regex[key] = re.compile(d[key])
print(end='\n')
def first_pass(self): def first_pass(self):
''' '''
...@@ -105,6 +104,8 @@ class FileExtractor: ...@@ -105,6 +104,8 @@ class FileExtractor:
with fileinput.input( with fileinput.input(
file, openhook=fileinput.hook_encoded("ISO-8859-1")) as f: file, openhook=fileinput.hook_encoded("ISO-8859-1")) as f:
lineData = 0 lineData = 0
lineHeader = 0
isHeader = True
filesRead += 1 filesRead += 1
for line in f: for line in f:
# header detection, skip header lines # header detection, skip header lines
...@@ -116,7 +117,7 @@ class FileExtractor: ...@@ -116,7 +117,7 @@ class FileExtractor:
elif 'endHeader' in self.__regex: elif 'endHeader' in self.__regex:
if self.__regex['endHeader'].match(line): if self.__regex['endHeader'].match(line):
lineHeader += 1 lineHeader += 1
isHeader = False isHeader = False
else: else:
lineHeader += 1 lineHeader += 1
continue continue
...@@ -163,12 +164,19 @@ class FileExtractor: ...@@ -163,12 +164,19 @@ class FileExtractor:
# initialize arrays, move at the end of firstPass ? # initialize arrays, move at the end of firstPass ?
for key in variables_1D: for key in variables_1D:
self.__data[key] = np.ones((self.n)) * self.__FillValue #self.__data[key] = np.ones((self.n)) * self.__FillValue
if '_FillValue' in self.roscop[key]:
self.__data[key] = np.full((self.n, self.roscop[key]['_FillValue']))
else:
self.__data[key] = np.empty(self.n)
for key in self.keys: for key in self.keys:
# mult by __fillValue next # mult by __fillValue next
# the shape parameter has to be an int or sequence of ints # the shape parameter has to be an int or sequence of ints
self.__data[key] = np.ones((self.n, self.m)) * self.__FillValue if '_FillValue' in self.roscop[key]:
self.__data[key] = np.full([self.n, self.m], self.roscop[key]['_FillValue'])
else:
self.__data[key] = np.empty([self.n, self.m])
for file in self.fname: for file in self.fname:
with fileinput.input( with fileinput.input(
...@@ -177,78 +185,87 @@ class FileExtractor: ...@@ -177,78 +185,87 @@ class FileExtractor:
for line in f: for line in f:
if f.filelineno() < self.lineHeader + 1: if f.filelineno() < self.lineHeader + 1:
# read and decode header # read and decode header
if self.__regex['DATETIME'].search(line): for k in self.__regex.keys():
(month, day, year, hour, minute, second) = \ # key is DATETIME
self.__regex['DATETIME'].search(line).groups() if k == "DATETIME" and self.__regex[k].search(line):
(month, day, year, hour, minute, second) = \
# format date and time to "May 09 2011 16:33:53" self.__regex[k].search(line).groups()
dateTime = "%s/%s/%s %s:%s:%s" % (day, month, year, hour, minute, second)
# format date and time to "May 09 2011 16:33:53"
# dateTime conversion to "09/05/2011 16:33:53" dateTime = "%s/%s/%s %s:%s:%s" % (day, month, year, hour, minute, second)
dateTime = "%s" % \
(dt.strptime(dateTime, "%d/%b/%Y %H:%M:%S").strftime("%d/%m/%Y %H:%M:%S")) # dateTime conversion to "09/05/2011 16:33:53"
# conversion to "20110509163353" dateTime = "%s" % \
epic_date = "%s" % \ (dt.strptime(dateTime, "%d/%b/%Y %H:%M:%S").strftime("%d/%m/%Y %H:%M:%S"))
(dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%Y%m%d%H%M%S")) # conversion to "20110509163353"
epic_date = "%s" % \
# conversion to julian day (dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%Y%m%d%H%M%S"))
julian = float((dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%j"))) \
+ ((float(hour) * 3600.) + (float(minute) * 60.) + float(second) ) / 86400. # conversion to julian day
julian = float((dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%j"))) \
# we use julian day with origine 0 + ((float(hour) * 3600.) + (float(minute) * 60.) + float(second) ) / 86400.
julian -= 1
print("{:07.4f} : {} / {}".format(julian, dateTime, epic_date)) # we use julian day with origine 0
self.__data['TIME'][n] = julian julian -= 1
self.__data['TIME'][n] = julian
if self.__regex['DATE'].search(line): # key is DATE
(month, day, year) = \ if k == "DATE" and self.__regex[k].search(line):
self.__regex['DATE'].search(line).groups() if device.lower() == 'ladcp':
if self.__regex['TIME'].search(line): (year, month, day) = \
(hour, minute, second) = \ self.__regex[k].search(line).groups()
self.__regex['TIME'].search(line).groups() else:
(month, day, year) = \
self.__regex[k].search(line).groups()
# key is TIME
if k == "TIME" and self.__regex[k].search(line):
(hour, minute, second) = \
self.__regex[k].search(line).groups()
# format date and time to "May 09 2011 16:33:53" # format date and time to "May 09 2011 16:33:53"
dateTime = "%s/%s/%s %s:%s:%s" % (day, month, year, hour, minute, second) dateTime = "%s/%s/%s %s:%s:%s" % (day, month, year, hour, minute, second)
# dateTime conversion to "09/05/2011 16:33:53" # dateTime conversion to "09/05/2011 16:33:53"
dateTime = "%s" % \ dateTime = "%s" % \
(dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%d/%m/%Y %H:%M:%S")) (dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%d/%m/%Y %H:%M:%S"))
# conversion to "20110509163353" # conversion to "20110509163353"
epic_date = "%s" % \ epic_date = "%s" % \
(dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%Y%m%d%H%M%S")) (dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%Y%m%d%H%M%S"))
# conversion to julian day # conversion to julian day
julian = float((dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%j"))) \ julian = float((dt.strptime(dateTime, "%d/%m/%Y %H:%M:%S").strftime("%j"))) \
+ ((float(hour) * 3600.) + (float(minute) * 60.) + float(second) ) / 86400. + ((float(hour) * 3600.) + (float(minute) * 60.) + float(second) ) / 86400.
# we use julian day with origine 0 # we use julian day with origine 0
julian -= 1 julian -= 1
print("{:07.4f} : {} / {}".format(julian, dateTime, epic_date)) self.__data['TIME'][n] = julian
self.__data['TIME'][n] = julian # key is LATITUDE
if k == "LATITUDE" and self.__regex[k].search(line):
if self.__regex['LATITUDE'].search(line): if device.lower() == 'ladcp':
(lat_deg, lat_min, lat_hemi) = self.__regex['LATITUDE'].search(line).groups() [latitude] = self.__regex[k].search(line).groups()
else:
# format latitude to string (lat_deg, lat_min, lat_hemi) = self.__regex[k].search(line).groups()
latitude_str = "%s%c%s %s" % (lat_deg, DEGREE, lat_min, lat_hemi)
# format latitude to string
# transform to decimal using ternary operator latitude_str = "%s%c%s %s" % (lat_deg, DEGREE, lat_min, lat_hemi)
latitude = float(lat_deg) + (float(lat_min) / 60.) if lat_hemi == 'N' else \
(float(lat_deg) + (float(lat_min) / 60.)) * -1 # transform to decimal using ternary operator
print("{:07.4f} : {}".format(latitude, latitude_str)) latitude = float(lat_deg) + (float(lat_min) / 60.) if lat_hemi == 'N' else \
self.__data['LATITUDE'][n] = latitude (float(lat_deg) + (float(lat_min) / 60.)) * -1
self.__data['LATITUDE'][n] = latitude
if self.__regex['LONGITUDE'].search(line): # key is LONGITUDE
(lon_deg, lon_min, lon_hemi) = self.__regex['LONGITUDE'].search(line).groups() if k == "LONGITUDE" and self.__regex[k].search(line):
if device.lower() == 'ladcp':
# format longitude to string [longitude] = self.__regex[k].search(line).groups()
longitude_str = "%s%c%s %s" % (lon_deg, DEGREE, lon_min, lon_hemi) else:
(lon_deg, lon_min, lon_hemi) = self.__regex[k].search(line).groups()
# transform to decimal using ternary operator
longitude = float(lon_deg) + (float(lon_min) / 60.) if lon_hemi == 'E' else \ # format longitude to string
(float(lon_deg) + (float(lon_min) / 60.)) * -1 longitude_str = "%s%c%s %s" % (lon_deg, DEGREE, lon_min, lon_hemi)
print("{:07.4f} : {}".format(longitude, longitude_str))
self.__data['LONGITUDE'][n] = longitude # transform to decimal using ternary operator
longitude = float(lon_deg) + (float(lon_min) / 60.) if lon_hemi == 'E' else \
(float(lon_deg) + (float(lon_min) / 60.)) * -1
self.__data['LONGITUDE'][n] = longitude
continue continue
# split the line, remove leading and trailing space before # split the line, remove leading and trailing space before
......
netcdf.py
+ 3
− 8
View file @ 9d288c59
import logging import logging
from netCDF4 import Dataset from netCDF4 import Dataset
from numpy import arange, dtype from numpy import arange, dtype
from physical_parameter import Roscop
def writeNetCDF(fileName, fe, variables_1D): def writeNetCDF(fileName, fe, r, variables_1D):
# ncvars is a dictionary that store a netcdf variable for each physical parameter key # ncvars is a dictionary that store a netcdf variable for each physical parameter key
ncvars = {} ncvars = {}
...@@ -14,9 +13,6 @@ def writeNetCDF(fileName, fe, variables_1D): ...@@ -14,9 +13,6 @@ def writeNetCDF(fileName, fe, variables_1D):
variables = variables_1D.copy() variables = variables_1D.copy()
dims_2D = ['TIME', 'DEPTH'] dims_2D = ['TIME', 'DEPTH']
# move to main after tests
r = Roscop("code_roscop.csv")
# create netcdf file # create netcdf file
nc = Dataset(fileName, "w", format="NETCDF3_CLASSIC") nc = Dataset(fileName, "w", format="NETCDF3_CLASSIC")
logging.debug(' ' + nc.data_model) logging.debug(' ' + nc.data_model)
...@@ -49,13 +45,14 @@ def writeNetCDF(fileName, fe, variables_1D): ...@@ -49,13 +45,14 @@ def writeNetCDF(fileName, fe, variables_1D):
hash.pop('_FillValue') hash.pop('_FillValue')
else: else:
fillvalue = None fillvalue = None
# create the variable # create the variable
if any(key in item for item in variables_1D): if any(key in item for item in variables_1D):
ncvars[key] = nc.createVariable( ncvars[key] = nc.createVariable(
key, dtype(hash['types']).char, (key,), fill_value=fillvalue) key, dtype(hash['types']).char, (key,), fill_value=fillvalue)
else: else:
ncvars[key] = nc.createVariable( ncvars[key] = nc.createVariable(
key, dtype(hash['types']).char, dims_2D, fill_value=fillvalue) key, dtype(hash['types']).char, dims_2D, fill_value=fillvalue)
# remove from the dictionary # remove from the dictionary
hash.pop('types') hash.pop('types')
# create dynamically variable attributes # create dynamically variable attributes
...@@ -71,10 +68,8 @@ def writeNetCDF(fileName, fe, variables_1D): ...@@ -71,10 +68,8 @@ def writeNetCDF(fileName, fe, variables_1D):
# write the ncvars # write the ncvars
for key in variables: for key in variables:
if any(key in item for item in variables_1D): if any(key in item for item in variables_1D):
#print("Key: {}, {}".format(key,fe[key]))
ncvars[key][:] = fe[key] ncvars[key][:] = fe[key]
else: else:
#print("Key: {}, {}".format(key,fe[key]))
ncvars[key][:, :] = fe[key] ncvars[key][:, :] = fe[key]
# close the netcdf file # close the netcdf file
......
oceano.py
+ 13
− 5
View file @ 9d288c59
...@@ -10,6 +10,7 @@ from configparser import ConfigParser ...@@ -10,6 +10,7 @@ from configparser import ConfigParser
import os import os
import distutils.util as du import distutils.util as du
import netcdf import netcdf
from physical_parameter import Roscop
# typeInstrument is a dictionary as key: files extension # typeInstrument is a dictionary as key: files extension
typeInstrument = {'CTD': ('cnv', 'CNV'), 'XBT': ( typeInstrument = {'CTD': ('cnv', 'CNV'), 'XBT': (
...@@ -21,6 +22,8 @@ filesBrowsePosition_column = 1 ...@@ -21,6 +22,8 @@ filesBrowsePosition_column = 1
# initialize filename use to save GUI configuration # initialize filename use to save GUI configuration
configfile = 'oceano.cfg' configfile = 'oceano.cfg'
# move to main after tests
r = Roscop("code_roscop.csv")
def processArgs(): def processArgs():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
...@@ -41,7 +44,7 @@ def processArgs(): ...@@ -41,7 +44,7 @@ def processArgs():
default='tests/test.toml') default='tests/test.toml')
parser.add_argument('-i', '--instrument', nargs='?', choices=ti.keys(), parser.add_argument('-i', '--instrument', nargs='?', choices=ti.keys(),
help='specify the instrument that produce files, eg CTD, XBT, TSG, LADCP') help='specify the instrument that produce files, eg CTD, XBT, TSG, LADCP')
parser.add_argument('-k', '--keys', nargs='+', default=['PRES', 'TEMP', 'PSAL'], parser.add_argument('-k', '--keys', nargs='+',
help='display dictionary for key(s), (default: %(default)s)') help='display dictionary for key(s), (default: %(default)s)')
parser.add_argument('-g', '--gui', action='store_true', parser.add_argument('-g', '--gui', action='store_true',
help='use GUI interface') help='use GUI interface')
...@@ -130,12 +133,17 @@ def process(args, cfg, ti): ...@@ -130,12 +133,17 @@ def process(args, cfg, ti):
n, m: array size n, m: array size
''' '''
print('processing...')
# check if no file selected or cancel button pressed # check if no file selected or cancel button pressed
logging.debug("File(s): {}, config: {}, Keys: {}".format( logging.debug("File(s): {}, config: {}, Keys: {}".format(
args.files, args.config, args.keys)) args.files, args.config, args.keys))
# fileExtractor # if physical parameters are not given from cmd line, option -k, use the toml <device>.split values
fe = FileExtractor(args.files, args.keys) if args.keys == None:
args.keys = cfg['split'][device.lower()].keys()
# extract header and data from files
fe = FileExtractor(args.files, r, args.keys)
# prepare (compile) each regular expression inside toml file under section [<device=ti>.header] # prepare (compile) each regular expression inside toml file under section [<device=ti>.header]
fe.set_regex(cfg, ti) fe.set_regex(cfg, ti)
...@@ -273,11 +281,11 @@ if __name__ == "__main__": ...@@ -273,11 +281,11 @@ if __name__ == "__main__":
parser.print_help(sys.stderr) parser.print_help(sys.stderr)
sys.exit(1) sys.exit(1)
keys = cfg['split'][device.lower()].keys()
# in command line mode (console) # in command line mode (console)
fe = process(args, cfg, device) fe = process(args, cfg, device)
#print("Dimensions: {} x {}".format(fe.m, fe.n)) #print("Dimensions: {} x {}".format(fe.m, fe.n))
#print(fe.disp()) #print(fe.disp())
# write the NetCDF file # write the NetCDF file
netcdf.writeNetCDF( "netcdf/OS_{}_{}.nc".format(cfg['cruise']['cycleMesure'], device), fe,variables_1D) ncfile = "netcdf/OS_{}_{}.nc".format(cfg['cruise']['cycleMesure'], device)
netcdf.writeNetCDF( ncfile, fe, r, variables_1D)
physical_parameter.py
+ 8
− 3
View file @ 9d288c59
...@@ -82,9 +82,14 @@ class Roscop: ...@@ -82,9 +82,14 @@ class Roscop:
# use the second line with key string to convert each numeric type into float # use the second line with key string to convert each numeric type into float
if theKey != 'string': if theKey != 'string':
if self['string'][k] == 'numeric': if self['string'][k] == 'numeric':
row[k] = float(row[k]) if 'float' in row['types']:
logging.debug( row[k] = float(row[k])
" {} -> {}, {} = {}".format(theKey, k, type(row[k]), row[k])) elif 'int' in row['types']:
row[k] = int(row[k])
else:
print('invalid type {}: {}'.format(theKey, k))
#logging.debug(
# " {} -> {}, {} = {}".format(theKey, k, type(row[k]), row[k]))
self.__hash[theKey] = row self.__hash[theKey] = row
return return
......
tests/test.toml
+ 19
− 14
View file @ 9d288c59
...@@ -24,7 +24,6 @@ instrumentNumber = "09P1263" ...@@ -24,7 +24,6 @@ instrumentNumber = "09P1263"
[ctd.header] [ctd.header]
isHeader = '^[*#]' isHeader = '^[*#]'
isDevice = '^\*\s+(Sea-Bird)'
DATETIME = 'System UpLoad Time\s*=\s*(\w+)\s+(\d+)\s+(\d+)\s+(\d+):(\d+):(\d+)' DATETIME = 'System UpLoad Time\s*=\s*(\w+)\s+(\d+)\s+(\d+)\s+(\d+):(\d+):(\d+)'
LATITUDE = 'NMEA\s+Latitude\s*[:=]\s*(\d+)\s+(\d+\.\d+)\s+(\w)' LATITUDE = 'NMEA\s+Latitude\s*[:=]\s*(\d+)\s+(\d+\.\d+)\s+(\w)'
LONGITUDE = 'NMEA\s+Longitude\s*[:=]\s*(\d+)\s+(\d+.\d+)\s+(\w)' LONGITUDE = 'NMEA\s+Longitude\s*[:=]\s*(\d+)\s+(\d+.\d+)\s+(\w)'
...@@ -82,7 +81,6 @@ comment = "Extract from .colcor files" ...@@ -82,7 +81,6 @@ comment = "Extract from .colcor files"
[ladcp] [ladcp]
cruisePrefix = "fr29" cruisePrefix = "fr29"
stationPrefixLength = 3 stationPrefixLength = 3
skipHeader = 10
acquisitionSoftware = "BBTALK" acquisitionSoftware = "BBTALK"
acquisitionVersion = 3.04 acquisitionVersion = 3.04
processingSoftware = "MATLAB-VISBECK" processingSoftware = "MATLAB-VISBECK"
...@@ -94,19 +92,26 @@ downSn = "24543" ...@@ -94,19 +92,26 @@ downSn = "24543"
title_summary = "IFM-GEOMAR/LDEO Matlab LADCP-Processing system adapted by FM/PR/JG" title_summary = "IFM-GEOMAR/LDEO Matlab LADCP-Processing system adapted by FM/PR/JG"
comment = "Extract from .lad files" comment = "Extract from .lad files"
[ladcp.header]
endHeader = 'Columns'
DATE = 'Date\s*=\s*(\d+)/(\d+)/(\d+)'
TIME = 'Start_Time\s*=\s*(\d+)[:hH](\d+):(\d+)'
LATITUDE = 'Latitude\s*[:=]\s*([-+]?\d+\.\d+)'
LONGITUDE = 'Longitude\s*[:=]\s*([-]?\d+.\d+)'
[split] [split]
[split.ctd] [split.ctd]
ETDD = 2 ETDD = 1
PRES = 3 PRES = 2
DEPTH = 4 DEPTH = 3
TEMP = 5 TEMP = 4
PSAL = 18 PSAL = 17
DENS = 20 DENS = 19
SVEL = 22 SVEL = 21
DOX2 = 16 DOX2 = 15
FLU2 = 14 FLU2 = 13
FLU3 = 15 FLU3 = 14
TUR3 = 13 TUR3 = 12
NAVG = 23 NAVG = 23
[split.ctdAll] [split.ctdAll]
...@@ -174,4 +179,4 @@ comment = "Extract from .lad files" ...@@ -174,4 +179,4 @@ comment = "Extract from .lad files"
DEPTH = 0 DEPTH = 0
SSJT = 1 SSJT = 1
SSTP = 2 SSTP = 2
SSPS = 3 SSPS = 3
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment