Skip to content
Snippets Groups Projects
Commit 7d197c69 authored by Jacques Grelet's avatar Jacques Grelet
Browse files

first version working with TSG Colcor

parent 66d90957
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
ascii.py
+ 11
− 4
View file @ 7d197c69
......@@ -152,12 +152,19 @@ def writeDataTrajectory(dataFile, cfg, device, fe, r):
cfg['cruise']['pi']))
# write header, second line with physical parameter liste, fill with N/A if necessary
f.write("DATE TIME LATITUDE LONGITUDE ")
for k in fe.keys:
f.write(f" {k} ")
for r in range(len(fe.keys),len(fe.variables_1D)+1):
f.write(' N/A ')
f.write(f" {k} ")
f.write("\n")
for n in range(fe.n):
for k in fe.keys:
if k == 'id':
fmt = fe.roscop['PROFILE']['format']
else:
fmt = fe.roscop[k]['format']
f.write(f" {fmt} " % (fe[k][n]))
f.write("\n")
f.close()
def writeTrajectory(cfg, device, fe, r):
if not os.path.exists(cfg['global']['ascii']):
......
netcdf.py
+ 113
− 0
View file @ 7d197c69
......@@ -118,3 +118,116 @@ def writeProfile(cfg, device, fe, r):
# close the netcdf file
nc.close()
print(' done...')
def writeTrajectory(cfg, device, fe, r):
# ncvars is a dictionary that store a netcdf variable for each physical parameter key
ncvars = {}
# variables and dimensions
variables = list(fe.getlist())
#print(variables)
variables.remove('id')
dims = ['time']
# create the output directory if it does not exist
if not os.path.exists(cfg['global']['netcdf']):
os.makedirs(cfg['global']['netcdf'])
# create netcdf file
fileName = "{}/OS_{}_{}.nc".format(cfg['global']
['netcdf'], cfg['cruise']['cycleMesure'], device)
print(f"writing netCDF file: {fileName}", end='', flush=True)
if not os.path.exists(cfg['global']['netcdf']):
os.makedirs(cfg['global']['netcdf'])
nc = Dataset(fileName, "w", format="NETCDF3_CLASSIC")
logging.debug(' ' + nc.data_model)
# create dimensions
# n is number of profiles, m the max size of profiles
time = nc.createDimension("time", fe.n)
# debug
logging.debug("time: {}".format(len(time)))
# create variables
# add dimensions before variables list
#for k in fe.keys:
# variables.append(k)
# variables.extend(fe.keys())
for key in variables:
# for each variables get the attributes dictionary from Roscop
hash = r[key]
# _FillValue attribute must be set when variable is created
# (using fill_value keyword to createVariable)
if '_FillValue' in hash:
fillvalue = hash['_FillValue']
# remove from the dictionary
hash.pop('_FillValue')
else:
fillvalue = None
# create the variable
ncvars[key] = nc.createVariable(
key, dtype(hash['types']).char, dims, fill_value=fillvalue)
""" if any(key in item for item in fe.variables_1D):
try:
# create variable whit same dimension name, as TIME(time)
ncvars[key] = nc.createVariable(
key, dtype(hash['types']).char, (key,), fill_value=fillvalue)
except:
# for BATH(time), it's a mess !
ncvars[key] = nc.createVariable(
key, dtype(hash['types']).char, 'time', fill_value=fillvalue)
else:
ncvars[key] = nc.createVariable(
key, dtype(hash['types']).char, dims, fill_value=fillvalue) """
# remove from the dictionary
hash.pop('types')
# create dynamically variable attributes
for k in hash.keys():
setattr(ncvars[key], k, hash[k])
nc._enddef()
# add global attributes
nc.data_type = "OceanSITES trajectory data"
nc.Conventions = "CF-1.7"
nc.title = cfg['global']['title']
nc.institution = cfg['global']['institution']
nc.source = cfg['global']['source']
nc.comment = cfg['global']['comment']
nc.references = cfg['global']['references']
nc.cycle_mesure = cfg['cruise']['cycleMesure']
nc.time_coverage_start = cfg['cruise']['beginDate']
nc.time_coverage_end = cfg['cruise']['endDate']
nc.timezone = cfg['cruise']['timezone']
nc.data_assembly_center = cfg['cruise']['institute']
nc.type_instrument = cfg[device.lower()]['typeInstrument']
nc.instrument_number = cfg[device.lower()]['instrumentNumber']
nc.date_update = datetime.today().strftime('%Y-%m-%dT%H:%M:%SZ')
nc.pi_name = cfg['cruise']['pi']
nc.processing_state = "1A"
nc.codification = "OOPC"
nc.format_version = "1.2"
nc.Netcdf_version = "3.6"
# debug
for key in variables:
logging.debug(" var: {}, dims: {}, shape: {}, dtype: {}, ndim: {}".format(
key, ncvars[key].dimensions, ncvars[key].shape, ncvars[key].dtype, ncvars[key].ndim))
# write the ncvars
for key in variables:
ncvars[key][:] = fe[key]
""" if any(key in item for item in fe.variables_1D):
ncvars[key][:] = fe[key]
else:
ncvars[key][:, :] = fe[key] """
# close the netcdf file
nc.close()
print(' done...')
oceano.py
+ 3
− 3
View file @ 7d197c69
......@@ -258,7 +258,7 @@ if __name__ == "__main__":
defaultRoscop = Path(cfg['global']['codeRoscop'])
if args.roscop != None:
defaultRoscop = args.roscop
r = Roscop(defaultRoscop)
roscop = Roscop(defaultRoscop)
# test arguments from sys.argv, args is never to None with default option set, without option
# or -g, call GUI
......@@ -300,9 +300,9 @@ if __name__ == "__main__":
print(type)
if type == 'PROFILE':
context = Profile(args.files, r, args.keys)
context = Profile(args.files, roscop, args.keys)
elif type == 'TRAJECTORY':
context = Trajectory(args.files, r, args.keys)
context = Trajectory(args.files, roscop, args.keys)
else:
print(f"Invalide type: {type}")
sys.exit()
......
physical_parameter.py
+ 3
− 3
View file @ 7d197c69
......@@ -129,9 +129,9 @@ if __name__ == "__main__":
# if args list is empty, key contain NoneType
if key is not None:
for k in key:
r[k]
print(f"{k}: {r[k]}")
print("{}: {}".format(key[0], r[key[0]]['long_name']))
""" print("{}: {}".format(key[0], r[key[0]]['long_name']))
r['TOTO'] = {'uncle': 'tata'}
print(r['TOTO'])
r['TEMP'] = 'tata'
r['TEMP'] = 'tata' """
trajectory.py
+ 30
− 59
View file @ 7d197c69
......@@ -5,6 +5,7 @@ import fileinput
import linecache
import logging
from operator import length_hint
import string
import toml
import sys
import argparse
......@@ -13,7 +14,7 @@ import re
from glob import glob
from datetime import datetime
import tools
from physical_parameter import Roscop
#from physical_parameter import Roscop
from notanorm import SqliteDb
import ascii
import netcdf
......@@ -23,13 +24,9 @@ import netcdf
table_data = """
CREATE TABLE data (
id INTEGER PRIMARY KEY,
date_time TEXT NOT NULL UNIQUE,
end_date_time TEXT,
julian_day REAL NOT NULL UNIQUE,
latitude REAL NOT NULL,
lat TEXT,
longitude REAL NOT NULL,
lon TEXT
DAYD REAL NOT NULL UNIQUE,
LATITUDE REAL NOT NULL,
LONGITUDE REAL NOT NULL
); """
class Trajectory:
......@@ -49,7 +46,6 @@ class Trajectory:
dbname: sqlite3 file, default i in memory
separator : str, column separator, default None (blank)
'''
variables_1D = ['PROFILE', 'TIME', 'LATITUDE', 'LONGITUDE','BATH']
def __init__(self, fname, roscop, keys, dbname=":memory:", separator=None):
'''constructor with values by default'''
......@@ -154,9 +150,6 @@ class Trajectory:
''' extract data from sqlite database and fill self.__data arrays
'''
# print infos after reding all files
hdr = self.db.query('SELECT * FROM data')
#st = self.db.query('SELECT COUNT(id) FROM data')
#print(f"SELECT COUNT({self.keys[0]}) FROM data")
n = self.db.count('data')
#m = self.db.max('data')
......@@ -165,49 +158,31 @@ class Trajectory:
#m = int(max_size[0][f"MAX({self.keys[0]})"])
print(f"Array sizes: {n}")
# initialize one dimension variables
for k in self.variables_1D:
#print(self.roscop[k])
if '_FillValue' in self.roscop[k]:
self.__data[k] = np.full(n, self.roscop[k]['_FillValue'])
else:
self.__data[k] = np.empty(n)
# get data from table station and fill array
#query = self.db.query('SELECT julian_day, latitude, longitude, bath FROM station')
query = self.db.select('data')
print(query)
'''
query = self.db.select('data', ['id', 'julian_day', 'end_date_time',
'latitude', 'longitude', 'bath'])
logging.debug(query)
profil_pk = []
# initialize one dimension variables
for idx, item in enumerate(query):
profil_pk.append(item['id'])
self.__data['PROFILE'][idx] = item['station']
#print(item['station'])
self.__data['TIME'][idx] = item['julian_day']
#self.__data['END_TIME'][idx] = item['end_date_time']
self.__data['LATITUDE'][idx] = item['latitude']
self.__data['LONGITUDE'][idx] = item['longitude']
self.__data['BATH'][idx] = item['bath']
# initialize array
for k in self.keys:
if '_FillValue' in self.roscop[k]:
self.__data[k] = np.full([n, m], self.roscop[k]['_FillValue'])
else:
self.__data[k] = np.empty([n, m])
# for each parameters
for k in self.keys:
# for each entries in station table, n is a list with indice start at 0
for i in profil_pk:
query = self.db.select('data', [k], station_id = profil_pk[i-1])
for idx, item in enumerate(query):
self.__data[k][i-1, idx] = item[k]
# define array size from table column name
for k in item:
# k is a type of <class 'notanorm.base.CIKey'>, convert to char !!!
key = f"{k}"
if idx == 0:
if key != 'id':
#print(f"{key}: {self.roscop[key]}")
if '_FillValue' in self.roscop[key]:
self.__data[key] = np.full(n, self.roscop[key]['_FillValue'])
else:
self.__data[key] = np.empty(n)
else:
self.__data[key] = np.empty(n)
# fill arrays
self.__data[key][idx] = item[key]
'''
self.n = n
# save all database columns as key
self.keys = self.__data.keys()
def read_files(self, cfg, device):
......@@ -272,7 +247,7 @@ class Trajectory:
latitude = float(lat_deg) + (float(lat_min) / 60.) if lat_hemi == 'N' else \
(float(lat_deg) + (float(lat_min) / 60.)) * -1
sql['LATITUDE'] = latitude
sql['lat'] = tools.Dec2dmc(float(latitude),'N')
#sql['lat'] = tools.Dec2dmc(float(latitude),'N')
if self.__regex['LONGITUDE'].search(line):
(lon_hemi, lon_deg, lon_min) = \
self.__regex['LONGITUDE'].search(line).groups()
......@@ -280,15 +255,15 @@ class Trajectory:
longitude = float(lon_deg) + (float(lon_min) / 60.) if lon_hemi == 'E' else \
(float(lon_deg) + (float(lon_min) / 60.)) * -1
sql['LONGITUDE'] = longitude
sql['lon'] = tools.Dec2dmc(float(longitude),'E')
#sql['lon'] = tools.Dec2dmc(float(longitude),'E')
# set datetime object
if 'dateTimeFormat' in cfg[device.lower()]:
dtf = cfg[device.lower()]['dateTimeFormat']
else:
dtf = "%d/%m/%Y %H:%M:%S"
sql['date_time'] = dt.strptime(dateTime, dtf)
sql['julian_day'] = tools.dt2julian(sql['date_time'])
date_time = dt.strptime(dateTime, dtf)
sql['DAYD'] = tools.dt2julian(date_time)
#print(f"Line: {line} separator: {self.__separator}")
# now, extract and process all data
......@@ -296,15 +271,11 @@ class Trajectory:
p = line.strip().split(self.__separator)
#print(p)
logging.debug(f"line split: {p}")
#logging.debug(f"line end: {p[-1]}")
# insert data from list p with indice hash[key]
#[sql[key] = p[hash[key]] for key in self.keys]
#sql['station_id'] = pk
for key in self.keys:
logging.debug(f"{key}, {hash[key]}, {p[hash[key]]}")
sql[key] = float(p[hash[key]])
#self.db.insert("data", station_id = 1, PRES = 1, TEMP = 20, PSAL = 35, DOX2 = 20, DENS = 30)
self.db.insert("data", sql )
# end of readline in file
......@@ -355,7 +326,7 @@ class Trajectory:
ascii.writeTrajectory(cfg, ti, self, self.roscop)
# write the NetCDF file
#netcdf.writeTrajectory(cfg, ti, self, self.roscop)
netcdf.writeTrajectory(cfg, ti, self, self.roscop)
# for testing in standalone context
# ---------------------------------
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment