Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
E
extract_cyclones
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Package Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
cyclobs
extract_cyclones
Commits
be4c5a61
Commit
be4c5a61
authored
Aug 24, 2020
by
CEVAER
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Refactored variables names
parent
9f605ed4
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
44 additions
and
42 deletions
+44
-42
bin/extractSmapCyclone.py
bin/extractSmapCyclone.py
+44
-42
No files found.
bin/extractSmapCyclone.py
View file @
be4c5a61
...
...
@@ -21,23 +21,23 @@ logger.setLevel(logging.INFO)
def
extract_date_from_filename
(
filename
):
date
P
art
=
"-"
.
join
(
filename
.
split
(
"_"
)[
4
:
7
])
return
datetime
.
datetime
.
strptime
(
date
P
art
,
"%Y-%m-%d"
)
date
_p
art
=
"-"
.
join
(
filename
.
split
(
"_"
)[
4
:
7
])
return
datetime
.
datetime
.
strptime
(
date
_p
art
,
"%Y-%m-%d"
)
def
get_track_points_from_database
(
session
,
smapDate
):
req
T
racks
=
session
.
query
(
SimpleTrack
,
func
.
ST_X
(
cast
(
SimpleTrack
.
geom
,
Geometry
)).
label
(
"lon"
),
def
get_track_points_from_database
(
session
,
smap
_
Date
):
req
_t
racks
=
session
.
query
(
SimpleTrack
,
func
.
ST_X
(
cast
(
SimpleTrack
.
geom
,
Geometry
)).
label
(
"lon"
),
func
.
ST_Y
(
cast
(
SimpleTrack
.
geom
,
Geometry
)).
label
(
"lat"
))
\
.
filter
(
and_
(
SimpleTrack
.
date
>=
smapDate
,
SimpleTrack
.
date
<
smapDate
+
datetime
.
timedelta
(
days
=
1
))).
group_by
(
SimpleTrack
).
all
()
return
req
T
racks
.
filter
(
and_
(
SimpleTrack
.
date
>=
smap
_
Date
,
SimpleTrack
.
date
<
smap
_
Date
+
datetime
.
timedelta
(
days
=
1
))).
group_by
(
SimpleTrack
).
all
()
return
req
_t
racks
def
get_colocated_track_points
(
dataset
,
track
Points
,
fileD
ate
,
deg_delta
=
3
,
time_delta
=
60
):
track
PointO
ffsets
=
[]
for
track
Point
in
trackP
oints
:
logger
.
debug
(
f
"Track point :
{
track
P
oint
}
"
)
sel
=
dataset
.
sel
(
lon
=
track
Point
[
"lon"
],
lat
=
trackP
oint
[
"lat"
],
method
=
"nearest"
)
def
get_colocated_track_points
(
dataset
,
track
_points
,
file_d
ate
,
deg_delta
=
3
,
time_delta
=
60
):
track
_point_o
ffsets
=
[]
for
track
_point
in
track_p
oints
:
logger
.
debug
(
f
"Track point :
{
track
_p
oint
}
"
)
sel
=
dataset
.
sel
(
lon
=
track
_point
[
"lon"
],
lat
=
track_p
oint
[
"lat"
],
method
=
"nearest"
)
logger
.
debug
(
f
"Point find : lon:
{
sel
[
'lon'
].
values
}
, lat:
{
sel
[
'lat'
].
values
}
, minute:
{
sel
[
'minute'
].
values
}
"
)
lonIndex
=
np
.
where
(
dataset
[
"lon"
].
data
==
sel
[
"lon"
].
data
)[
0
][
0
]
...
...
@@ -50,50 +50,52 @@ def get_colocated_track_points(dataset, trackPoints, fileDate, deg_delta=3, time
# The time offset is stored with all the track point data to ease later processing
for
node
in
df
.
index
:
if
not
pd
.
isna
(
df
[
"minute"
][
node
]):
point
Date
=
fileD
ate
+
df
[
"minute"
][
node
]
time
Offset
=
abs
(
trackPoint
[
"date"
]
-
pointD
ate
)
logger
.
debug
(
f
" Data point date:
{
point
Date
}
, Track point date:
{
trackP
oint
[
'date'
]
}
,"
f
" Time offset:
{
time
O
ffset
}
"
)
point
_date
=
file_d
ate
+
df
[
"minute"
][
node
]
time
_offset
=
abs
(
track_point
[
"date"
]
-
point_d
ate
)
logger
.
debug
(
f
" Data point date:
{
point
_date
}
, Track point date:
{
track_p
oint
[
'date'
]
}
,"
f
" Time offset:
{
time
_o
ffset
}
"
)
# If this is the first node processed OR that the previous nodes had NAN times OR that previous nodes
# offset times are greater than this one
if
node
==
0
or
not
"time
Offset"
in
trackPoint
or
trackPoint
[
"timeOffset"
]
>
timeO
ffset
:
track
Point
[
"timeOffset"
]
=
timeO
ffset
track
P
oint
[
"node"
]
=
node
track
P
oint
[
"lonIndex"
]
=
lonIndex
track
P
oint
[
"latIndex"
]
=
latIndex
if
node
==
0
or
not
"time
_offset"
in
track_point
or
track_point
[
"time_offset"
]
>
time_o
ffset
:
track
_point
[
"time_offset"
]
=
time_o
ffset
track
_p
oint
[
"node"
]
=
node
track
_p
oint
[
"lonIndex"
]
=
lonIndex
track
_p
oint
[
"latIndex"
]
=
latIndex
logger
.
debug
(
f
"ISNOTNA"
)
if
"time
Offset"
in
trackP
oint
:
track
PointOffsets
.
append
(
trackP
oint
)
if
"time
_offset"
in
track_p
oint
:
track
_point_offsets
.
append
(
track_p
oint
)
kept
TrackP
oints
=
{}
kept
_track_p
oints
=
{}
# Getting the best track point for each sid (storm id).
# One SMAP file can contain several cyclones acquisitions so that is why we are doing this filtering per sid
for
track
Point
in
trackPointO
ffsets
:
for
track
_point
in
track_point_o
ffsets
:
# TODO replace timedelta with database track sampling time
if
track
Point
[
"timeO
ffset"
]
<=
datetime
.
timedelta
(
minutes
=
7
,
seconds
=
30
):
if
track
Point
[
"sid"
]
not
in
keptTrackP
oints
:
kept
TrackPoints
[
trackPoint
[
"sid"
]]
=
trackP
oint
elif
kept
TrackPoints
[
trackPoint
[
"sid"
]][
"timeOffset"
]
>
trackPoint
[
"timeO
ffset"
]:
kept
TrackPoints
[
trackPoint
[
"sid"
]]
=
trackP
oint
if
track
_point
[
"time_o
ffset"
]
<=
datetime
.
timedelta
(
minutes
=
7
,
seconds
=
30
):
if
track
_point
[
"sid"
]
not
in
kept_track_p
oints
:
kept
_track_points
[
track_point
[
"sid"
]]
=
track_p
oint
elif
kept
_track_points
[
track_point
[
"sid"
]][
"time_offset"
]
>
track_point
[
"time_o
ffset"
]:
kept
_track_points
[
track_point
[
"sid"
]]
=
track_p
oint
return
keptTrackPoints
return
kept_track_points
#def extract
def
process_smap_file
(
session
,
file
):
logger
.
debug
(
f
"Processing
{
file
}
..."
)
filename
=
os
.
path
.
basename
(
file
)
file
D
ate
=
extract_date_from_filename
(
filename
)
logger
.
debug
(
f
"File date
{
file
D
ate
}
"
)
track
Points
=
get_track_points_from_database
(
session
,
fileD
ate
)
logger
.
debug
(
f
"Number of track point found :
{
len
(
track
P
oints
)
}
"
)
file
_d
ate
=
extract_date_from_filename
(
filename
)
logger
.
debug
(
f
"File date
{
file
_d
ate
}
"
)
track
_points
=
get_track_points_from_database
(
session
,
file_d
ate
)
logger
.
debug
(
f
"Number of track point found :
{
len
(
track
_p
oints
)
}
"
)
track
Points
=
[{
"sid"
:
trackPoint
[
0
].
sid
,
"lon"
:
shape360
(
trackP
oint
.
lon
,
0
)[
0
],
"lat"
:
track
P
oint
.
lat
,
"date"
:
track
Point
[
0
].
date
}
for
trackPoint
in
trackP
oints
]
track
_points
=
[{
"sid"
:
track_point
[
0
].
sid
,
"lon"
:
shape360
(
track_p
oint
.
lon
,
0
)[
0
],
"lat"
:
track
_p
oint
.
lat
,
"date"
:
track
_point
[
0
].
date
}
for
track_point
in
track_p
oints
]
dataset
=
xarray
.
open_dataset
(
file
)
kept
TrackPoints
=
get_colocated_track_points
(
dataset
,
trackPoints
,
fileD
ate
)
logger
.
info
(
f
"For file
{
filename
}
Kept track that will be used to extract SMAP data:
{
kept
TrackP
oints
}
"
)
kept
_track_points
=
get_colocated_track_points
(
dataset
,
track_points
,
file_d
ate
)
logger
.
info
(
f
"For file
{
filename
}
Kept track that will be used to extract SMAP data:
{
kept
_track_p
oints
}
"
)
if
__name__
==
"__main__"
:
...
...
@@ -115,7 +117,7 @@ if __name__ == "__main__":
engine
=
create_engine
(
args
.
dbd
)
Session
=
sessionmaker
(
bind
=
engine
)
smap
T
ests
=
[
smap
_t
ests
=
[
"/home/datawork-cersat-public/provider/remss/satellite/l3/smap/smap/wind/v1.0/daily/2020/205/RSS_smap_wind_daily_2020_07_23_v01.0.nc"
,
"/home/datawork-cersat-public/provider/remss/satellite/l3/smap/smap/wind/v1.0/daily/2020/206/RSS_smap_wind_daily_2020_07_24_v01.0.nc"
,
"/home/datawork-cersat-public/provider/remss/satellite/l3/smap/smap/wind/v1.0/daily/2020/207/RSS_smap_wind_daily_2020_07_25_v01.0.nc"
,
...
...
@@ -124,5 +126,5 @@ if __name__ == "__main__":
"/home/datawork-cersat-public/provider/remss/satellite/l3/smap/smap/wind/v1.0/daily/2020/210/RSS_smap_wind_daily_2020_07_28_v01.0.nc"
,
"/home/datawork-cersat-public/provider/remss/satellite/l3/smap/smap/wind/v1.0/daily/2020/211/RSS_smap_wind_daily_2020_07_29_v01.0.nc"
,
]
for
f
in
smap
T
ests
:
for
f
in
smap
_t
ests
:
process_smap_file
(
Session
(),
f
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment