Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
E
extract_cyclones
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Merge Requests
0
Merge Requests
0
Operations
Operations
Incidents
Packages & Registries
Packages & Registries
Package Registry
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
cyclobs
extract_cyclones
Commits
d2d1009a
Commit
d2d1009a
authored
Nov 09, 2020
by
CEVAER
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Modified SMOS, SMAP attributes. Not writing files that have below 5 pixels.
parent
42e5e429
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
60 additions
and
16 deletions
+60
-16
bin/extractSmapCyclone.py
bin/extractSmapCyclone.py
+16
-3
bin/extractSmosCyclone.py
bin/extractSmosCyclone.py
+15
-3
extract_cyclones/__init__.py
extract_cyclones/__init__.py
+29
-10
No files found.
bin/extractSmapCyclone.py
View file @
d2d1009a
...
...
@@ -50,14 +50,27 @@ if __name__ == "__main__":
engine
=
create_engine
(
args
.
dbd
,
pool_size
=
50
,
max_overflow
=
0
)
Session
=
sessionmaker
(
bind
=
engine
)
attrs_to_del
=
[
"aprrox_local_equatorial_crossing_time"
,
"swath_sector"
,
"first_orbit"
,
"last_orbit"
]
#
attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "first_orbit", "last_orbit"]
var_attr_edit
=
{
"lon"
:
{
"valid_min"
:
-
180
,
"valid_max"
:
180
}}
#var_attr_edit = {"lon": {"valid_min": -180, "valid_max": 180}}
attrs
=
{
"Conventions"
:
"CF-1.6"
,
"title"
:
"SMAP ocean surface wind speed (10m above surface) cropped around Tropical Cyclone track"
,
"institution"
:
"IFREMER/LOPS"
,
"sourceProduct"
:
""
,
"sourceReference"
:
"""Meissner, T., L. Ricciardulli, and
F. Wentz, 2018: Remote Sensing Systems SMAP daily Sea Surface Winds
Speeds on 0.25 deg grid, Version 01.0. [NRT or FINAL]. Remote Sensing
Systems, Santa Rosa, CA. Available online at
www.remss.com/missions/smap/"""
,
"missionName"
:
"SMAP"
}
for
f
in
args
.
input
:
# TODO generate filename_format depending on input filename
process_file
(
Session
(),
file
=
f
,
output_path
=
args
.
output
,
extract_date_func
=
extract_date_from_filename
,
attrs_to_del
=
attrs_to_del
,
var_attr_edit
=
var_attr_edit
,
var_to_del
=
[
"minute"
],
var_to_del
=
[
"minute"
],
attrs
=
attrs
,
wind_col
=
"wind"
,
time_col
=
"minute"
,
lat_col
=
"lat"
,
lon_col
=
"lon"
,
pass_col
=
"node"
,
pass_width
=
1000
,
...
...
bin/extractSmosCyclone.py
View file @
d2d1009a
...
...
@@ -81,15 +81,27 @@ if __name__ == "__main__":
Session
=
sessionmaker
(
bind
=
engine
)
# Attributes that'll be deleted
attrs_to_del
=
[
"aprrox_local_equatorial_crossing_time"
,
"swath_sector"
,
"geospatial_bounds"
,
"geospatial_lat_min"
,
"geospatial_lat_max"
,
"geospatial_lon_min"
,
"geospatial_lon_max"
]
#attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
# "geospatial_lat_min", "geospatial_lat_max", "geospatial_lon_min", "geospatial_lon_max"]
attrs
=
{
"Conventions"
:
"CF-1.6"
,
"title"
:
"SMOS ocean surface wind speed cropped around Tropical Cyclone track"
,
"institution"
:
"IFREMER/LOPS"
,
"sourceProduct"
:
""
,
"sourceReference"
:
"""Reul Nicolas, Tenerelli Joseph, Chapron Bertrand, Vandemark Doug, Quilfen
Yves, Kerr Yann (2012). SMOS satellite L-band radiometer: A new capability for ocean surface
remote sensing in hurricanes. Journal Of Geophysical Research-oceans, 117. Publisher's official version :
http://dx.doi.org/10.1029/2011JC007474"""
,
"missionName"
:
"SMOS"
}
from
extract_cyclones
import
process_file
for
f
in
args
.
input
:
# TODO generate filename_format depending on input filename
process_file
(
Session
(),
file
=
f
,
output_path
=
args
.
output
,
extract_date_func
=
extract_date_from_filename
,
attrs_to_del
=
attrs_to_del
,
var_to_del
=
[
"measurement_time"
],
wind_col
=
"wind_speed"
,
var_to_del
=
[
"measurement_time"
],
wind_col
=
"wind_speed"
,
attrs
=
attrs
,
time_col
=
"measurement_time"
,
lat_col
=
"lat"
,
lon_col
=
"lon"
,
pass_col
=
None
,
pass_width
=
1200
,
filename_format
=
"SM_OPER_MIR_<time>_<sid>.nc"
,
specfic_func
=
set_nan_low_quality
)
extract_cyclones/__init__.py
View file @
d2d1009a
...
...
@@ -164,7 +164,6 @@ def compute_footprint(dataset, wind_field):
df
=
dataset
.
to_dataframe
()
df
=
df
[
pd
.
isna
(
df
[
wind_field
])
==
False
]
df
=
df
.
reset_index
()
gdf
=
gpd
.
GeoDataFrame
(
df
,
geometry
=
gpd
.
points_from_xy
(
df
.
lon
,
df
.
lat
))
...
...
@@ -184,13 +183,14 @@ def extract_start_stop_measure(dataset, time_col_name):
def
extract_write_cyclone_data
(
dataset
,
kept_track_points
,
filename
,
output_path
,
file_date
,
attrs_to_del
,
var_to_del
,
lon_col_name
,
file_date
,
attrs
,
attrs
_to_del
,
var_to_del
,
lon_col_name
,
lat_col_name
,
wind_col_name
,
time_col_name
,
pass_col_name
,
filename_format
,
is_full_time
,
extract_size_km
=
2000
,
var_attr_edit
=
{},
specific_func
=
None
):
specific_func
=
None
,
min_pixel
=
5
):
for
sid
,
track_point
in
kept_track_points
.
items
():
# Km per deg in latitude
km_per_deg_lat
=
111
...
...
@@ -253,6 +253,13 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
if
specific_func
is
not
None
:
sel
=
specific_func
(
sel
)
# if there are less than x points, remove the image
df
=
sel
.
to_dataframe
()
df
=
df
[
pd
.
isna
(
df
[
wind_col_name
])
==
False
]
if
len
(
df
.
index
)
<=
min_pixel
:
logger
.
info
(
f
"Data has less than
{
min_pixel
}
pixel of data. Not writing to file."
)
return
# Extracting min and max time to set attributes on NetCDF
min_time
,
max_time
=
extract_start_stop_measure
(
sel
,
time_col_name
)
if
max_time
-
min_time
>
datetime
.
timedelta
(
minutes
=
30
):
...
...
@@ -261,18 +268,29 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
logger
.
debug
(
f
"min_time:
{
min_time
}
, max_time:
{
max_time
}
"
)
if
is_full_time
:
sel
.
attrs
[
"measurement_start_date"
]
=
min_time
.
strftime
(
"%Y-%m-%d %H:%M:%S"
)
sel
.
attrs
[
"measurement_stop_date"
]
=
max_time
.
strftime
(
"%Y-%m-%d %H:%M:%S"
)
#sel.attrs["measurement_start_date"] = min_time.strftime("%Y-%m-%d %H:%M:%S")
#sel.attrs["measurement_stop_date"] = max_time.strftime("%Y-%m-%d %H:%M:%S")
sel
.
attrs
[
"measurementDate"
]
=
(
min_time
+
(
max_time
-
min_time
)
/
2
).
strftime
(
"%Y-%m-%d %H:%M:%S"
)
year
=
str
(
min_time
.
year
)
doy
=
min_time
.
strftime
(
'%j'
)
else
:
min_date
=
file_date
+
min_time
max_date
=
file_date
+
max_time
year
=
min_date
.
year
doy
=
min_date
.
strftime
(
'%j'
)
sel
.
attrs
[
"measurement_start_date"
]
=
str
(
min_date
)
sel
.
attrs
[
"measurement_stop_date"
]
=
str
(
file_date
+
max_time
)
#sel.attrs["measurement_start_date"] = str(min_date)
#sel.attrs["measurement_stop_date"] = str(file_date + max_time)
sel
.
attrs
[
"measurementDate"
]
=
(
min_date
+
(
max_date
-
min_date
)
/
2
).
strftime
(
"%Y-%m-%d %H:%M:%S"
)
# Deleting all attributes
sel
.
attrs
.
clear
()
# Setting new attributes
for
attr
in
attrs
:
sel
.
attrs
[
attr
]
=
attrs
[
attr
]
# Removing attributes
# FIXME potentially to remove because shadowed by the above deletion
for
attr
in
attrs_to_del
:
if
attr
in
sel
.
attrs
:
del
sel
.
attrs
[
attr
]
...
...
@@ -313,8 +331,9 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
sel
.
to_netcdf
(
os
.
path
.
join
(
out
,
output_filename
),
format
=
"NETCDF4_CLASSIC"
)
def
process_file
(
session
,
file
,
output_path
,
extract_date_func
,
attrs_to_del
,
var_to_del
,
wind_col
,
time_col
,
lat_col
,
lon_col
,
filename_format
,
pass_width
,
pass_col
=
None
,
var_attr_edit
=
{},
def
process_file
(
session
,
file
,
output_path
,
extract_date_func
,
attrs
,
var_to_del
,
wind_col
,
time_col
,
lat_col
,
lon_col
,
filename_format
,
pass_width
,
pass_col
=
None
,
attrs_to_del
=
[],
var_attr_edit
=
{},
specfic_func
=
None
):
logger
.
info
(
f
"Processing
{
file
}
..."
)
...
...
@@ -341,7 +360,7 @@ def process_file(session, file, output_path, extract_date_func, attrs_to_del, va
extract_write_cyclone_data
(
dataset
,
kept_track_points
,
filename
,
output_path
,
file_date
,
wind_col_name
=
wind_col
,
time_col_name
=
time_col
,
lat_col_name
=
lat_col
,
lon_col_name
=
lon_col
,
pass_col_name
=
pass_col
,
attrs_to_del
=
attrs_to_del
,
var_to_del
=
var_to_del
,
pass_col_name
=
pass_col
,
attrs
=
attrs
,
attrs
_to_del
=
attrs_to_del
,
var_to_del
=
var_to_del
,
filename_format
=
filename_format
,
is_full_time
=
full_time
,
extract_size_km
=
pass_width
*
2
,
var_attr_edit
=
var_attr_edit
,
specific_func
=
specfic_func
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment