oceano.py

#!/usr/bin/env python

import argparse
import sys
import re
import PySimpleGUI as sg
import toml
import logging
from profile import Profile
from trajectory import Trajectory
from pathlib import Path
from configparser import ConfigParser
from glob import glob
import distutils.util as du
from physical_parameter import Roscop

EXIT_SUCCESS = 0
EXIT_FAILURE = 1

dict_type = {'PROFILE': ['CTD', 'BTL','XBT','LADCP'], 'TRAJECTORY': ['TSG','MTO']}

# typeInstrument is a dictionary as key: files extension
typeInstrument = {'CTD': ('cnv', 'CNV'), 'XBT': (
    'EDF', 'edf'), 'LADCP': ('lad', 'LAD'), 'TSG': ('colcor','COLCOR'),
    'BTL': ('btl', 'BTL')}
#variables_1D = ['TIME', 'LATITUDE', 'LONGITUDE','BATH']
ti = typeInstrument  # an alias
filesBrowsePosition_row = 2
filesBrowsePosition_column = 1

# initialize filename use to save GUI configuration
configfile = 'oceano.cfg'
# default file name for ROSCOP csv
defaultRoscop = 'code_roscop.csv'

# example, PIRATA cruises:
# FR30:
# python oceano.py /m/PIRATA-FR30/data-processing/CTD/data/cnv/d*.cnv -i CTD -k PRES TEMP PSAL DOX2 FLU2 -r code_roscop.csv


def processArgs():
    parser = argparse.ArgumentParser(
        description='This program read multiple ASCII file, extract physical parameter \
                    following ROSCOP codification at the given column, fill arrays, write header file ',
        usage='\npython oceano.py data/CTD/cnv/dfr2900[1-3].cnv -i CTD -d\n'
        'python oceano.py data/CTD/cnv/dfr2900[1-3].cnv -i CTD -k PRES TEMP PSAL DOX2 DENS\n'
        'python oceano.py data/CTD/cnv/dfr29*.cnv -i CTD -d\n'
        'python oceano.py data/XBT/T7_0000*.EDF -i XBT -k DEPTH TEMP SVEL\n'
        'python oceano.py data/LADCP/*.lad -i LADCP -k DEPTH EWCT NSCT\n'
        'python oceano.py data/CTD/btl/fr290*.btl -i BTL -k BOTL PRES DEPTH ETDD TE01 TE02 PSA1 PSA2 DO11 DO12 DO21 DO22 FLU2'
        ' \n',
        epilog='J. Grelet IRD US191 - March 2019 / Feb 2020')
    parser.add_argument('-d', '--debug', help='display debug informations',
                        action='store_true')
    parser.add_argument('--demo', nargs='?', choices=ti.keys(),
                        help='specify the commande line for instrument, eg CTD, XBT, TSG, LADCP')
    parser.add_argument('-c', '--config', help="toml configuration file, (default: %(default)s)",
                        default='config.toml')
    parser.add_argument('-i', '--instrument', nargs='?', choices=ti.keys(),
                        help='specify the instrument that produce files, eg CTD, XBT, TSG, LADCP')
    parser.add_argument('-r', '--roscop', nargs='?',
                        help='specify location and ROSCOP file name, (default: code_roscop.csv)')
    parser.add_argument('-k', '--keys', nargs='+',
                        help='display dictionary for key(s), (default: %(default)s)')
    parser.add_argument('-g', '--gui', action='store_true',
                        help='use GUI interface')
    parser.add_argument('--database', help='save sqlite3 database in test.db instead of memory',
                        action='store_true')
    # type=argparse.FileType('r') don't work with under DOS 
    parser.add_argument('files', nargs='*',
                        help='ASCII file(s) to parse')                        
    return parser

# TODOS:
# DEPTH is missing
# file name is not clear at startup
# if no file selected, don't leave the program


def defineGUI():
    '''
    function to define and create the graphical interface, written with PySimpleGUI
    '''
    # check if GUI config file exist
    if args.instrument != None:
        instrument_default_value = args.instrument
    else:
        instrument_default_value = 'CTD'

    # get all devices
    devices = list(ti.keys())

    # change look and feel color scheme
    sg.ChangeLookAndFeel('SandyBeach')
    frameLayout = {}

    # define a frame layout for each instrument (device) with composite key as PRES_CTD, TEMP_XBT
    for d in devices:
        keys = cfg['split'][d.lower()].keys()
        # List comprehensions
        frameLayout[d] = [[sg.Checkbox(k, key='{:s}_{:s}'.format(k, d),
                                       tooltip='Select the extract the physical parameter {}'.format(k))] for k in keys]

    # define GUI layout
    layout = ([[sg.Text('File(s) to read and convert')],                              # row 0
               [sg.Multiline(size=(40, 5), key='_IN_'),                               # row 1, col 0
                sg.Input(key='_HIDDEN_', visible=False,                               # row 1, col 1
                         enable_events=True),
                sg.FilesBrowse(key='_HIDDEN_', initial_folder=None,                   # row 1, col 2
                               tooltip='Choose one or more files')],
               [sg.Combo(list(ti.keys()), enable_events=True, size=(8, 1),          # row 2
                         default_value=instrument_default_value,
                         key='_DEVICE_', tooltip='Select the instrument')],
               [sg.Frame(d, frameLayout[d], key='_FRAME_{:s}'.format(d), visible=True)          # row 3
                for d in devices],
               [sg.OK(), sg.CloseButton('Cancel')]])                                  # row 4
    # [sg.CloseButton('Run'), sg.CloseButton('Cancel')]])

    # create a local instance windows used to reload the saved config from file
    window = sg.Window('Oceano converter', layout)
    window.Finalize()
    window.LoadFromDisk(configfile)

    return window


def updateFilesBrowseCombo(window, extentions, x, y):
    '''
    special function used to update the FilesBrowseCombo with canvas poisition
    instead of key because the same key is assign to shadow input object
    '''
    e = window.Rows[x][y]    # hardcoded
    e.FileTypes = []         # init to empty list
    for ext in extentions:
        e.FileTypes.append(("{} files".format(ext), "*.{}".format(ext)))

    e.initial_folder = 'data/{}'.format(extentions[0])
    window.Finalize

def process_gui():
          # setup the GUI windows Layout
        window = defineGUI()
        device = window.find_element('_DEVICE_').DefaultValue
        keys = cfg['split'][device.lower()].keys()

        # can't update combo with FindElement('_HIDDEN_').Update(), we use this function
        # with hardcoded FilesBrowseCombo position
        updateFilesBrowseCombo(window,
            ti[device], filesBrowsePosition_column, filesBrowsePosition_row)

        # set the rigth frame for device visible, dosn't work
        #  File "C:\git\python\PySimpleGUI\PySimpleGUI.py", line 2362, in Update
        #  self.TKFrame.pack()
        #  AttributeError: 'NoneType' object has no attribute 'pack'
        # for d in list(ti.keys()):
        #     print(d)
        #     if d == device:
        #         window.FindElement(
        #             '_FRAME_{:s}'.format(d)).Update(visible=True)

        # main GUI loop
        while True:
           # display the main windows
            event, values = window.Read()

            #print(event, values)

            if event == 'Cancel' or event == None:
                raise SystemExit("Cancelling: user exit")

            if event == 'OK':  # end of initialization, process data now
                # values['_HIDDEN_'] is a string with files separated by ';' and Profile need a list
                files = values['_HIDDEN_'].split(';')
                args.files = files

                # test if a or more file are selected
                if not all(args.files):
                    sg.Popup("Cancel", "No filename supplied")
                    # raise SystemExit("Cancelling: no filename supplied")
                    continue
                break

            if event == '_DEVICE_':
                # you have to go into the bowels of the pygi code, to get the instance of the Combo
                # by the line and column number of the window to update its "fileType" property.
                device = values['_DEVICE_']
                updateFilesBrowseCombo(window,
                    ti[device], filesBrowsePosition_column, filesBrowsePosition_row)
                # TODOS: reset checkbox

            # update the Multilines instance from FilesBrowse return
            if event == '_HIDDEN_':
                window.Element('_IN_').Update(
                    values['_HIDDEN_'].split(';'))

        # save program configuration
        window.SaveToDisk(configfile)

        # debug return values from GUI
        logging.debug("Event: {}, Values: {}".format(event, values))

        # extract selected parameters (true) from dict values only for selected device
        args.keys = []
        for k in values.keys():
            if k[0] == '_' or values[k] == False:
                continue
            # check if device == device
            if not device in k:
                continue
            else:
                args.keys.append(re.sub('_\w+$', '', k))

        '''
        # process of files start here
        fe = process(args, cfg, device)

        # display result in popup GUI
        dims = "Dimensions: {} x {}".format(fe.n, fe.m)
        sg.PopupScrolled('Oceano2python', dims,
                         fe.disp(),  size=(80, 40))
        '''
        # It will output to a debug window. Bug ? debug windows xas closed before exiting program
        # print = sg.Print
        # or
        # print = sg.Print(size=(80,40))

        return window, fe, device



if __name__ == "__main__":
    '''
    usage:
    > python oceano.py data/CTD/cnv/dfr2900[1-3].cnv -i CTD
    > python oceano.py data/CTD/cnv/dfr29*.cnv -i CTD -k PRES TEMP PSAL DOX2 DENS -d
    > python oceano.py data/CTD/cnv/dfr2900[1-3].cnv -i CTD -k PRES TEMP PSAL DOX2 DENS
    > python oceano.py data/CTD/cnv/dfr29*.cnv -i CTD -d
    > python oceano.py data/XBT/T7_0000*.EDF -i XBT -k DEPTH TEMP SVEL
    > python oceano.py data/LADCP/*.lad -i LADCP - k DEPTH EWCT NSCT
    > python oceano.py data/CTD/btl/fr290*.btl -i BTL -k BOTL PRES DEPTH ETDD TE01 TE02 PSA1 PSA2 DO11 DO12 DO21 DO22 FLU2
    '''
    # recover and process line arguments
    parser = processArgs()
    args = parser.parse_args()

    # set looging mode if debug
    if args.debug:
        logging.basicConfig(
            format='%(levelname)s:%(message)s', level=logging.DEBUG)

    # read config Toml file and get the physical parameter list (Roscop code) for the specified instrument
    cfg = toml.load(args.config)
    # this the select device from command line !
    device = str(args.instrument)  # convert one element list to str

    # get roscop file
    if cfg['global']['codeRoscop'] != None:
        defaultRoscop = Path(cfg['global']['codeRoscop'])
    if args.roscop != None:
        defaultRoscop = args.roscop
    roscop = Roscop(defaultRoscop)

    # test arguments from sys.argv, args is never to None with default option set, without option
    # or -g, call GUI
    if args.gui or len(sys.argv) == 1:
        # define graphical interface
        window, fe, device = process_gui()

    else:
        # demo mode, only in command line
        if args.demo != None:
            print('demo mode: {}'.format(args.demo))
            sys.exit(1)
        # test if a or more file are selected
        else:
            if args.files == []:
                print(
                    'Error, you need to specify one or more files to process !!!', end='\n\n')
                parser.print_help(sys.stderr)
                sys.exit(1)
            
            # work with DOs, Git bash and Linux
            files = []
            for file in args.files:  
                files += glob(file)  
            args.files = files
   
        if device == 'None':
            print(
                'Error: missing option -i or --instrument, instrument = {}\n'.format(device))
            parser.print_help(sys.stderr)
            sys.exit(1)

    def search_dict(dict_type, instrument):
        for k,i in dict_type.items():
            if instrument in i:
                return(k)

    type = search_dict(dict_type, args.instrument)
    print(type)

    if type == 'PROFILE':
        context = Profile(args.files, roscop, args.keys)
    elif type == 'TRAJECTORY':
        context = Trajectory(args.files, roscop, args.keys)
    else:
        print(f"Invalide type: {type}")
        sys.exit()
        
    context.process(args, cfg, device)


    # write ASCII hdr and data files
    #ascii.writeAscii(cfg, device, fe, r)

    # write the NetCDF file
    #netcdf_profile.writeNetCDF(cfg, device, fe, r)