From d024f3ac7a4b18c3e2540c1e2fab969ae586ba57 Mon Sep 17 00:00:00 2001
From: Jacques Grelet <jacques.grelet@gmail.cpm>
Date: Fri, 22 Apr 2022 16:10:44 -0300
Subject: [PATCH] add update_table data
---
file_extractor.py | 58 +++++++++++++++++++++++++++++++++++------------
1 file changed, 44 insertions(+), 14 deletions(-)
diff --git a/file_extractor.py b/file_extractor.py
index c6898b7..bcfc5bb 100644
--- a/file_extractor.py
+++ b/file_extractor.py
@@ -56,9 +56,11 @@ class FileExtractor:
Parameters
----------
- fname : file, str, pathlib.Path, list of str
- File, filename, or list to read.
+ fname : files, str, pathlib.Path, list of str
+ File, filename, or list to read and process.
+ roscop: file which describe physical parameter (code_roscop.csv)
keys: list of physical parameter to extract
+ dbname: sqlite3 file, default i in memory
separator : str, column separator, default None (blank)
'''
variables_1D = ['PROFILE', 'TIME', 'LATITUDE', 'LONGITUDE','BATH']
@@ -108,6 +110,14 @@ class FileExtractor:
''' overload string representation '''
return 'Class FileExtractor, file: %s, size = %d x %d' % (self.fname, self.n, self.m)
+ def update_table(self, keys):
+ ''' update table data and add new column from pm (physical parameter)'''
+ print(keys, type(keys))
+ for pm in keys:
+ print(f"\tUpdate table data with new column {pm}")
+ addColumn = f"ALTER TABLE data ADD COLUMN {pm} REAL NOT NULL"
+ self.db.query(addColumn)
+
def create_tables(self):
''' Create table station and data'''
self.db.query("DROP TABLE IF EXISTS station")
@@ -117,11 +127,8 @@ class FileExtractor:
# Create table data
self.db.query(table_data)
- # update table data and add new column from pm (physical parameter)
- for pm in self.keys:
- #print(f"\tUpdate table data with new column {pm}")
- addColumn = f"ALTER TABLE data ADD COLUMN {pm} REAL NOT NULL"
- self.db.query(addColumn)
+ # update table
+ self.update_table(self.keys)
def close(self):
self.db.close()
@@ -383,6 +390,7 @@ class FileExtractor:
continue
sql = {}
+ # insert data from list p with indice hash[key]
#[sql[key] = p[hash[key]] for key in self.keys]
sql['station_id'] = pk
for key in self.keys:
@@ -414,9 +422,17 @@ class FileExtractor:
if __name__ == "__main__":
# usage:
- # > python file_extractor.py data/CTD/cnv/dfr2900[1-3].cnv -i CTD -d
- # > python file_extractor.py data/CTD/cnv/dfr2900*.cnv -k PRES ETDD TEMP PSAL DOX2 DENS -i CTD
- # > python file_extractor.py data/XBT/T7_0000*.EDF -k DEPTH TEMP SVEL -i XBT
+ # python file_extractor.py data/CTD/cnv/dfr2900[1-3].cnv -i CTD -d
+ # python file_extractor.py data/CTD/cnv/dfr2900*.cnv -i CTD -k PRES ETDD TEMP PSAL DOX2 DENS
+ # python file_extractor.py data/XBT/T7_0000*.EDF -k DEPTH TEMP SVEL -i XBT
+ # python file_extractor.py data/CTD/btl/fr290*.btl -i BTL -k BOTL DEPTH ETDD TE01 PSA1 DO11
+
+ # typeInstrument is a dictionary as key: files extension
+ typeInstrument = {'CTD': ('cnv', 'CNV'), 'XBT': (
+ 'EDF', 'edf'), 'LADCP': ('lad', 'LAD'), 'TSG': ('colcor','COLCOR'),
+ 'BTL': ('btl', 'BTL')}
+ #variables_1D = ['TIME', 'LATITUDE', 'LONGITUDE','BATH']
+ ti = typeInstrument # an alias
parser = argparse.ArgumentParser(
description='This class read multiple ASCII file, extract physical parameter \
@@ -426,12 +442,14 @@ if __name__ == "__main__":
action='store_true')
parser.add_argument('-c', '--config', help="toml configuration file, (default: %(default)s)",
default='config.toml')
- parser.add_argument('-i', '--instrument', nargs='?', choices=['CTD','XBT','LADCP'],
+ parser.add_argument('-i', '--instrument', nargs='?', choices=ti.keys(),
help='specify the instrument that produce files, eg CTD, XBT, TSG, LADCP')
parser.add_argument('-k', '--keys', nargs='+', default=['PRES', 'TEMP', 'PSAL'],
help='display dictionary for key(s), (default: %(default)s)')
parser.add_argument('files', nargs='*',
help='ASCII file(s) to parse')
+ parser.add_argument('--sbe35', nargs='*',
+ help='ASCII file(s) to parse')
# display extra logging info
# see: https://stackoverflow.com/questions/14097061/easier-way-to-enable-verbose-logging
@@ -441,6 +459,7 @@ if __name__ == "__main__":
logging.basicConfig(
format='%(levelname)s:%(message)s', level=logging.DEBUG)
+
# work with DOs, Git bash and Linux
files = []
for file in args.files:
@@ -455,9 +474,20 @@ if __name__ == "__main__":
fe.set_regex(cfg, args.instrument, 'header')
fe.read_files(cfg, args.instrument)
logging.debug(f"Indices: {fe.n} x {fe.m}\nkeys: {fe.keys}")
+ # if args.sbe35 and args.instrument == 'BTL':
+ # sbe35 = []
+ # for t in args.sbe35:
+ # sbe35 += glob(t)
+ # fe.fname = sbe35
+ # fe.set_regex(cfg, args.instrument, 'header')
+ # fe.read_files(cfg, args.instrument)
+ # elif args.sbe35 and args.instrument != 'BTL':
+ # print("option --sbe35 can only be used with the BTL instrument (-i BTL)")
+ # exit
+
# # debug
- logging.debug(fe['PRES'])
- logging.debug(fe['TEMP'][0][1])
logging.debug(fe.getlist())
- logging.debug(fe['PSAL'])
+ for k in fe.keys:
+ for i in range(fe.n):
+ logging.debug(f"{fe[k][i][1]} : {fe[k][i][fe.m-1]}")
fe.close()
--
GitLab