Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
O
Oceano2python
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
US191
Oceano2python
Commits
7d197c69
Commit
7d197c69
authored
2 years ago
by
Jacques Grelet
Browse files
Options
Downloads
Patches
Plain Diff
first version working with TSG Colcor
parent
66d90957
No related branches found
Branches containing commit
No related tags found
No related merge requests found
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
ascii.py
+11
-4
11 additions, 4 deletions
ascii.py
netcdf.py
+113
-0
113 additions, 0 deletions
netcdf.py
oceano.py
+3
-3
3 additions, 3 deletions
oceano.py
physical_parameter.py
+3
-3
3 additions, 3 deletions
physical_parameter.py
trajectory.py
+30
-59
30 additions, 59 deletions
trajectory.py
with
160 additions
and
69 deletions
ascii.py
+
11
−
4
View file @
7d197c69
...
...
@@ -152,12 +152,19 @@ def writeDataTrajectory(dataFile, cfg, device, fe, r):
cfg
[
'
cruise
'
][
'
pi
'
]))
# write header, second line with physical parameter liste, fill with N/A if necessary
f
.
write
(
"
DATE TIME LATITUDE LONGITUDE
"
)
for
k
in
fe
.
keys
:
f
.
write
(
f
"
{
k
}
"
)
for
r
in
range
(
len
(
fe
.
keys
),
len
(
fe
.
variables_1D
)
+
1
):
f
.
write
(
'
N/A
'
)
f
.
write
(
f
"
{
k
}
"
)
f
.
write
(
"
\n
"
)
for
n
in
range
(
fe
.
n
):
for
k
in
fe
.
keys
:
if
k
==
'
id
'
:
fmt
=
fe
.
roscop
[
'
PROFILE
'
][
'
format
'
]
else
:
fmt
=
fe
.
roscop
[
k
][
'
format
'
]
f
.
write
(
f
"
{
fmt
}
"
%
(
fe
[
k
][
n
]))
f
.
write
(
"
\n
"
)
f
.
close
()
def
writeTrajectory
(
cfg
,
device
,
fe
,
r
):
if
not
os
.
path
.
exists
(
cfg
[
'
global
'
][
'
ascii
'
]):
...
...
This diff is collapsed.
Click to expand it.
netcdf.py
+
113
−
0
View file @
7d197c69
...
...
@@ -118,3 +118,116 @@ def writeProfile(cfg, device, fe, r):
# close the netcdf file
nc
.
close
()
print
(
'
done...
'
)
def
writeTrajectory
(
cfg
,
device
,
fe
,
r
):
# ncvars is a dictionary that store a netcdf variable for each physical parameter key
ncvars
=
{}
# variables and dimensions
variables
=
list
(
fe
.
getlist
())
#print(variables)
variables
.
remove
(
'
id
'
)
dims
=
[
'
time
'
]
# create the output directory if it does not exist
if
not
os
.
path
.
exists
(
cfg
[
'
global
'
][
'
netcdf
'
]):
os
.
makedirs
(
cfg
[
'
global
'
][
'
netcdf
'
])
# create netcdf file
fileName
=
"
{}/OS_{}_{}.nc
"
.
format
(
cfg
[
'
global
'
]
[
'
netcdf
'
],
cfg
[
'
cruise
'
][
'
cycleMesure
'
],
device
)
print
(
f
"
writing netCDF file:
{
fileName
}
"
,
end
=
''
,
flush
=
True
)
if
not
os
.
path
.
exists
(
cfg
[
'
global
'
][
'
netcdf
'
]):
os
.
makedirs
(
cfg
[
'
global
'
][
'
netcdf
'
])
nc
=
Dataset
(
fileName
,
"
w
"
,
format
=
"
NETCDF3_CLASSIC
"
)
logging
.
debug
(
'
'
+
nc
.
data_model
)
# create dimensions
# n is number of profiles, m the max size of profiles
time
=
nc
.
createDimension
(
"
time
"
,
fe
.
n
)
# debug
logging
.
debug
(
"
time: {}
"
.
format
(
len
(
time
)))
# create variables
# add dimensions before variables list
#for k in fe.keys:
# variables.append(k)
# variables.extend(fe.keys())
for
key
in
variables
:
# for each variables get the attributes dictionary from Roscop
hash
=
r
[
key
]
# _FillValue attribute must be set when variable is created
# (using fill_value keyword to createVariable)
if
'
_FillValue
'
in
hash
:
fillvalue
=
hash
[
'
_FillValue
'
]
# remove from the dictionary
hash
.
pop
(
'
_FillValue
'
)
else
:
fillvalue
=
None
# create the variable
ncvars
[
key
]
=
nc
.
createVariable
(
key
,
dtype
(
hash
[
'
types
'
]).
char
,
dims
,
fill_value
=
fillvalue
)
"""
if any(key in item for item in fe.variables_1D):
try:
# create variable whit same dimension name, as TIME(time)
ncvars[key] = nc.createVariable(
key, dtype(hash[
'
types
'
]).char, (key,), fill_value=fillvalue)
except:
# for BATH(time), it
'
s a mess !
ncvars[key] = nc.createVariable(
key, dtype(hash[
'
types
'
]).char,
'
time
'
, fill_value=fillvalue)
else:
ncvars[key] = nc.createVariable(
key, dtype(hash[
'
types
'
]).char, dims, fill_value=fillvalue)
"""
# remove from the dictionary
hash
.
pop
(
'
types
'
)
# create dynamically variable attributes
for
k
in
hash
.
keys
():
setattr
(
ncvars
[
key
],
k
,
hash
[
k
])
nc
.
_enddef
()
# add global attributes
nc
.
data_type
=
"
OceanSITES trajectory data
"
nc
.
Conventions
=
"
CF-1.7
"
nc
.
title
=
cfg
[
'
global
'
][
'
title
'
]
nc
.
institution
=
cfg
[
'
global
'
][
'
institution
'
]
nc
.
source
=
cfg
[
'
global
'
][
'
source
'
]
nc
.
comment
=
cfg
[
'
global
'
][
'
comment
'
]
nc
.
references
=
cfg
[
'
global
'
][
'
references
'
]
nc
.
cycle_mesure
=
cfg
[
'
cruise
'
][
'
cycleMesure
'
]
nc
.
time_coverage_start
=
cfg
[
'
cruise
'
][
'
beginDate
'
]
nc
.
time_coverage_end
=
cfg
[
'
cruise
'
][
'
endDate
'
]
nc
.
timezone
=
cfg
[
'
cruise
'
][
'
timezone
'
]
nc
.
data_assembly_center
=
cfg
[
'
cruise
'
][
'
institute
'
]
nc
.
type_instrument
=
cfg
[
device
.
lower
()][
'
typeInstrument
'
]
nc
.
instrument_number
=
cfg
[
device
.
lower
()][
'
instrumentNumber
'
]
nc
.
date_update
=
datetime
.
today
().
strftime
(
'
%Y-%m-%dT%H:%M:%SZ
'
)
nc
.
pi_name
=
cfg
[
'
cruise
'
][
'
pi
'
]
nc
.
processing_state
=
"
1A
"
nc
.
codification
=
"
OOPC
"
nc
.
format_version
=
"
1.2
"
nc
.
Netcdf_version
=
"
3.6
"
# debug
for
key
in
variables
:
logging
.
debug
(
"
var: {}, dims: {}, shape: {}, dtype: {}, ndim: {}
"
.
format
(
key
,
ncvars
[
key
].
dimensions
,
ncvars
[
key
].
shape
,
ncvars
[
key
].
dtype
,
ncvars
[
key
].
ndim
))
# write the ncvars
for
key
in
variables
:
ncvars
[
key
][:]
=
fe
[
key
]
"""
if any(key in item for item in fe.variables_1D):
ncvars[key][:] = fe[key]
else:
ncvars[key][:, :] = fe[key]
"""
# close the netcdf file
nc
.
close
()
print
(
'
done...
'
)
This diff is collapsed.
Click to expand it.
oceano.py
+
3
−
3
View file @
7d197c69
...
...
@@ -258,7 +258,7 @@ if __name__ == "__main__":
defaultRoscop
=
Path
(
cfg
[
'
global
'
][
'
codeRoscop
'
])
if
args
.
roscop
!=
None
:
defaultRoscop
=
args
.
roscop
r
=
Roscop
(
defaultRoscop
)
r
oscop
=
Roscop
(
defaultRoscop
)
# test arguments from sys.argv, args is never to None with default option set, without option
# or -g, call GUI
...
...
@@ -300,9 +300,9 @@ if __name__ == "__main__":
print
(
type
)
if
type
==
'
PROFILE
'
:
context
=
Profile
(
args
.
files
,
r
,
args
.
keys
)
context
=
Profile
(
args
.
files
,
r
oscop
,
args
.
keys
)
elif
type
==
'
TRAJECTORY
'
:
context
=
Trajectory
(
args
.
files
,
r
,
args
.
keys
)
context
=
Trajectory
(
args
.
files
,
r
oscop
,
args
.
keys
)
else
:
print
(
f
"
Invalide type:
{
type
}
"
)
sys
.
exit
()
...
...
This diff is collapsed.
Click to expand it.
physical_parameter.py
+
3
−
3
View file @
7d197c69
...
...
@@ -129,9 +129,9 @@ if __name__ == "__main__":
# if args list is empty, key contain NoneType
if
key
is
not
None
:
for
k
in
key
:
r
[
k
]
print
(
f
"
{
k
}
:
{
r
[
k
]
}
"
)
print
(
"
{}: {}
"
.
format
(
key
[
0
],
r
[
key
[
0
]][
'
long_name
'
]))
"""
print(
"
{}: {}
"
.format(key[0], r[key[0]][
'
long_name
'
]))
r[
'
TOTO
'
] = {
'
uncle
'
:
'
tata
'
}
print(r[
'
TOTO
'
])
r
[
'
TEMP
'
]
=
'
tata
'
r[
'
TEMP
'
] =
'
tata
'
"""
This diff is collapsed.
Click to expand it.
trajectory.py
+
30
−
59
View file @
7d197c69
...
...
@@ -5,6 +5,7 @@ import fileinput
import
linecache
import
logging
from
operator
import
length_hint
import
string
import
toml
import
sys
import
argparse
...
...
@@ -13,7 +14,7 @@ import re
from
glob
import
glob
from
datetime
import
datetime
import
tools
from
physical_parameter
import
Roscop
#
from physical_parameter import Roscop
from
notanorm
import
SqliteDb
import
ascii
import
netcdf
...
...
@@ -23,13 +24,9 @@ import netcdf
table_data
=
"""
CREATE TABLE data (
id INTEGER PRIMARY KEY,
date_time TEXT NOT NULL UNIQUE,
end_date_time TEXT,
julian_day REAL NOT NULL UNIQUE,
latitude REAL NOT NULL,
lat TEXT,
longitude REAL NOT NULL,
lon TEXT
DAYD REAL NOT NULL UNIQUE,
LATITUDE REAL NOT NULL,
LONGITUDE REAL NOT NULL
);
"""
class
Trajectory
:
...
...
@@ -49,7 +46,6 @@ class Trajectory:
dbname: sqlite3 file, default i in memory
separator : str, column separator, default None (blank)
'''
variables_1D
=
[
'
PROFILE
'
,
'
TIME
'
,
'
LATITUDE
'
,
'
LONGITUDE
'
,
'
BATH
'
]
def
__init__
(
self
,
fname
,
roscop
,
keys
,
dbname
=
"
:memory:
"
,
separator
=
None
):
'''
constructor with values by default
'''
...
...
@@ -154,9 +150,6 @@ class Trajectory:
'''
extract data from sqlite database and fill self.__data arrays
'''
# print infos after reding all files
hdr
=
self
.
db
.
query
(
'
SELECT * FROM data
'
)
#st = self.db.query('SELECT COUNT(id) FROM data')
#print(f"SELECT COUNT({self.keys[0]}) FROM data")
n
=
self
.
db
.
count
(
'
data
'
)
#m = self.db.max('data')
...
...
@@ -165,49 +158,31 @@ class Trajectory:
#m = int(max_size[0][f"MAX({self.keys[0]})"])
print
(
f
"
Array sizes:
{
n
}
"
)
# initialize one dimension variables
for
k
in
self
.
variables_1D
:
#print(self.roscop[k])
if
'
_FillValue
'
in
self
.
roscop
[
k
]:
self
.
__data
[
k
]
=
np
.
full
(
n
,
self
.
roscop
[
k
][
'
_FillValue
'
])
else
:
self
.
__data
[
k
]
=
np
.
empty
(
n
)
# get data from table station and fill array
#query = self.db.query('SELECT julian_day, latitude, longitude, bath FROM station')
query
=
self
.
db
.
select
(
'
data
'
)
print
(
query
)
'''
query = self.db.select(
'
data
'
, [
'
id
'
,
'
julian_day
'
,
'
end_date_time
'
,
'
latitude
'
,
'
longitude
'
,
'
bath
'
])
logging.debug(query)
profil_pk = []
# initialize one dimension variables
for
idx
,
item
in
enumerate
(
query
):
profil_pk.append(item[
'
id
'
])
self.__data[
'
PROFILE
'
][idx] = item[
'
station
'
]
#print(item[
'
station
'
])
self.__data[
'
TIME
'
][idx] = item[
'
julian_day
'
]
#self.__data[
'
END_TIME
'
][idx] = item[
'
end_date_time
'
]
self.__data[
'
LATITUDE
'
][idx] = item[
'
latitude
'
]
self.__data[
'
LONGITUDE
'
][idx] = item[
'
longitude
'
]
self.__data[
'
BATH
'
][idx] = item[
'
bath
'
]
# initialize array
for k in self.keys:
if
'
_FillValue
'
in self.roscop[k]:
self.__data[k] = np.full([n, m], self.roscop[k][
'
_FillValue
'
])
else:
self.__data[k] = np.empty([n, m])
# for each parameters
for k in self.keys:
# for each entries in station table, n is a list with indice start at 0
for i in profil_pk:
query = self.db.select(
'
data
'
, [k], station_id = profil_pk[i-1])
for idx, item in enumerate(query):
self.__data[k][i-1, idx] = item[k]
# define array size from table column name
for
k
in
item
:
# k is a type of <class 'notanorm.base.CIKey'>, convert to char !!!
key
=
f
"
{
k
}
"
if
idx
==
0
:
if
key
!=
'
id
'
:
#print(f"{key}: {self.roscop[key]}")
if
'
_FillValue
'
in
self
.
roscop
[
key
]:
self
.
__data
[
key
]
=
np
.
full
(
n
,
self
.
roscop
[
key
][
'
_FillValue
'
])
else
:
self
.
__data
[
key
]
=
np
.
empty
(
n
)
else
:
self
.
__data
[
key
]
=
np
.
empty
(
n
)
# fill arrays
self
.
__data
[
key
][
idx
]
=
item
[
key
]
'''
self
.
n
=
n
# save all database columns as key
self
.
keys
=
self
.
__data
.
keys
()
def
read_files
(
self
,
cfg
,
device
):
...
...
@@ -272,7 +247,7 @@ class Trajectory:
latitude
=
float
(
lat_deg
)
+
(
float
(
lat_min
)
/
60.
)
if
lat_hemi
==
'
N
'
else
\
(
float
(
lat_deg
)
+
(
float
(
lat_min
)
/
60.
))
*
-
1
sql
[
'
LATITUDE
'
]
=
latitude
sql
[
'
lat
'
]
=
tools
.
Dec2dmc
(
float
(
latitude
),
'
N
'
)
#
sql['lat'] = tools.Dec2dmc(float(latitude),'N')
if
self
.
__regex
[
'
LONGITUDE
'
].
search
(
line
):
(
lon_hemi
,
lon_deg
,
lon_min
)
=
\
self
.
__regex
[
'
LONGITUDE
'
].
search
(
line
).
groups
()
...
...
@@ -280,15 +255,15 @@ class Trajectory:
longitude
=
float
(
lon_deg
)
+
(
float
(
lon_min
)
/
60.
)
if
lon_hemi
==
'
E
'
else
\
(
float
(
lon_deg
)
+
(
float
(
lon_min
)
/
60.
))
*
-
1
sql
[
'
LONGITUDE
'
]
=
longitude
sql
[
'
lon
'
]
=
tools
.
Dec2dmc
(
float
(
longitude
),
'
E
'
)
#
sql['lon'] = tools.Dec2dmc(float(longitude),'E')
# set datetime object
if
'
dateTimeFormat
'
in
cfg
[
device
.
lower
()]:
dtf
=
cfg
[
device
.
lower
()][
'
dateTimeFormat
'
]
else
:
dtf
=
"
%d/%m/%Y %H:%M:%S
"
sql
[
'
date_time
'
]
=
dt
.
strptime
(
dateTime
,
dtf
)
sql
[
'
julian_day
'
]
=
tools
.
dt2julian
(
sql
[
'
date_time
'
]
)
date_time
=
dt
.
strptime
(
dateTime
,
dtf
)
sql
[
'
DAYD
'
]
=
tools
.
dt2julian
(
date_time
)
#print(f"Line: {line} separator: {self.__separator}")
# now, extract and process all data
...
...
@@ -296,15 +271,11 @@ class Trajectory:
p
=
line
.
strip
().
split
(
self
.
__separator
)
#print(p)
logging
.
debug
(
f
"
line split:
{
p
}
"
)
#logging.debug(f"line end: {p[-1]}")
# insert data from list p with indice hash[key]
#[sql[key] = p[hash[key]] for key in self.keys]
#sql['station_id'] = pk
for
key
in
self
.
keys
:
logging
.
debug
(
f
"
{
key
}
,
{
hash
[
key
]
}
,
{
p
[
hash
[
key
]]
}
"
)
sql
[
key
]
=
float
(
p
[
hash
[
key
]])
#self.db.insert("data", station_id = 1, PRES = 1, TEMP = 20, PSAL = 35, DOX2 = 20, DENS = 30)
self
.
db
.
insert
(
"
data
"
,
sql
)
# end of readline in file
...
...
@@ -355,7 +326,7 @@ class Trajectory:
ascii
.
writeTrajectory
(
cfg
,
ti
,
self
,
self
.
roscop
)
# write the NetCDF file
#
netcdf.writeTrajectory(cfg, ti, self, self.roscop)
netcdf
.
writeTrajectory
(
cfg
,
ti
,
self
,
self
.
roscop
)
# for testing in standalone context
# ---------------------------------
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment