Commit d2d1009a authored by CEVAER's avatar CEVAER

Modified SMOS, SMAP attributes. Not writing files that have below 5 pixels.

parent 42e5e429
......@@ -50,14 +50,27 @@ if __name__ == "__main__":
engine = create_engine(args.dbd, pool_size=50, max_overflow=0)
Session = sessionmaker(bind=engine)
attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "first_orbit", "last_orbit"]
#attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "first_orbit", "last_orbit"]
var_attr_edit = {"lon": {"valid_min": -180, "valid_max": 180}}
#var_attr_edit = {"lon": {"valid_min": -180, "valid_max": 180}}
attrs = {"Conventions": "CF-1.6",
"title": "SMAP ocean surface wind speed (10m above surface) cropped around Tropical Cyclone track",
"institution": "IFREMER/LOPS",
"sourceProduct": "",
"sourceReference": """Meissner, T., L. Ricciardulli, and
F. Wentz, 2018: Remote Sensing Systems SMAP daily Sea Surface Winds
Speeds on 0.25 deg grid, Version 01.0. [NRT or FINAL]. Remote Sensing
Systems, Santa Rosa, CA. Available online at
www.remss.com/missions/smap/""",
"missionName": "SMAP"
}
for f in args.input:
# TODO generate filename_format depending on input filename
process_file(Session(), file=f, output_path=args.output, extract_date_func=extract_date_from_filename,
attrs_to_del=attrs_to_del, var_attr_edit=var_attr_edit, var_to_del=["minute"],
var_to_del=["minute"],
attrs=attrs,
wind_col="wind", time_col="minute",
lat_col="lat",
lon_col="lon", pass_col="node", pass_width=1000,
......
......@@ -81,15 +81,27 @@ if __name__ == "__main__":
Session = sessionmaker(bind=engine)
# Attributes that'll be deleted
attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
"geospatial_lat_min", "geospatial_lat_max", "geospatial_lon_min", "geospatial_lon_max"]
#attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
# "geospatial_lat_min", "geospatial_lat_max", "geospatial_lon_min", "geospatial_lon_max"]
attrs = {"Conventions": "CF-1.6",
"title": "SMOS ocean surface wind speed cropped around Tropical Cyclone track",
"institution": "IFREMER/LOPS",
"sourceProduct": "",
"sourceReference": """Reul Nicolas, Tenerelli Joseph, Chapron Bertrand, Vandemark Doug, Quilfen
Yves, Kerr Yann (2012). SMOS satellite L-band radiometer: A new capability for ocean surface
remote sensing in hurricanes. Journal Of Geophysical Research-oceans, 117. Publisher's official version :
http://dx.doi.org/10.1029/2011JC007474""",
"missionName": "SMOS"
}
from extract_cyclones import process_file
for f in args.input:
# TODO generate filename_format depending on input filename
process_file(Session(), file=f, output_path=args.output, extract_date_func=extract_date_from_filename,
attrs_to_del=attrs_to_del, var_to_del=["measurement_time"], wind_col="wind_speed",
var_to_del=["measurement_time"], wind_col="wind_speed",
attrs=attrs,
time_col="measurement_time",
lat_col="lat", lon_col="lon", pass_col=None, pass_width=1200,
filename_format="SM_OPER_MIR_<time>_<sid>.nc", specfic_func=set_nan_low_quality)
......@@ -164,7 +164,6 @@ def compute_footprint(dataset, wind_field):
df = dataset.to_dataframe()
df = df[pd.isna(df[wind_field]) == False]
df = df.reset_index()
gdf = gpd.GeoDataFrame(
df, geometry=gpd.points_from_xy(df.lon, df.lat))
......@@ -184,13 +183,14 @@ def extract_start_stop_measure(dataset, time_col_name):
def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path,
file_date, attrs_to_del, var_to_del, lon_col_name,
file_date, attrs, attrs_to_del, var_to_del, lon_col_name,
lat_col_name, wind_col_name, time_col_name, pass_col_name,
filename_format,
is_full_time,
extract_size_km=2000,
var_attr_edit={},
specific_func=None):
specific_func=None,
min_pixel=5):
for sid, track_point in kept_track_points.items():
# Km per deg in latitude
km_per_deg_lat = 111
......@@ -253,6 +253,13 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
if specific_func is not None:
sel = specific_func(sel)
# if there are less than x points, remove the image
df = sel.to_dataframe()
df = df[pd.isna(df[wind_col_name]) == False]
if len(df.index) <= min_pixel:
logger.info(f"Data has less than {min_pixel} pixel of data. Not writing to file.")
return
# Extracting min and max time to set attributes on NetCDF
min_time, max_time = extract_start_stop_measure(sel, time_col_name)
if max_time - min_time > datetime.timedelta(minutes=30):
......@@ -261,18 +268,29 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
logger.debug(f"min_time: {min_time}, max_time: {max_time}")
if is_full_time:
sel.attrs["measurement_start_date"] = min_time.strftime("%Y-%m-%d %H:%M:%S")
sel.attrs["measurement_stop_date"] = max_time.strftime("%Y-%m-%d %H:%M:%S")
#sel.attrs["measurement_start_date"] = min_time.strftime("%Y-%m-%d %H:%M:%S")
#sel.attrs["measurement_stop_date"] = max_time.strftime("%Y-%m-%d %H:%M:%S")
sel.attrs["measurementDate"] = (min_time + (max_time - min_time) / 2).strftime("%Y-%m-%d %H:%M:%S")
year = str(min_time.year)
doy = min_time.strftime('%j')
else:
min_date = file_date + min_time
max_date = file_date + max_time
year = min_date.year
doy = min_date.strftime('%j')
sel.attrs["measurement_start_date"] = str(min_date)
sel.attrs["measurement_stop_date"] = str(file_date + max_time)
#sel.attrs["measurement_start_date"] = str(min_date)
#sel.attrs["measurement_stop_date"] = str(file_date + max_time)
sel.attrs["measurementDate"] = (min_date + (max_date - min_date) / 2).strftime("%Y-%m-%d %H:%M:%S")
# Deleting all attributes
sel.attrs.clear()
# Setting new attributes
for attr in attrs:
sel.attrs[attr] = attrs[attr]
# Removing attributes
# FIXME potentially to remove because shadowed by the above deletion
for attr in attrs_to_del:
if attr in sel.attrs:
del sel.attrs[attr]
......@@ -313,8 +331,9 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
sel.to_netcdf(os.path.join(out, output_filename), format="NETCDF4_CLASSIC")
def process_file(session, file, output_path, extract_date_func, attrs_to_del, var_to_del, wind_col,
time_col, lat_col, lon_col, filename_format, pass_width, pass_col=None, var_attr_edit={},
def process_file(session, file, output_path, extract_date_func, attrs, var_to_del, wind_col,
time_col, lat_col, lon_col, filename_format, pass_width, pass_col=None, attrs_to_del=[],
var_attr_edit={},
specfic_func=None):
logger.info(f"Processing {file}...")
......@@ -341,7 +360,7 @@ def process_file(session, file, output_path, extract_date_func, attrs_to_del, va
extract_write_cyclone_data(dataset, kept_track_points, filename, output_path, file_date, wind_col_name=wind_col,
time_col_name=time_col, lat_col_name=lat_col, lon_col_name=lon_col,
pass_col_name=pass_col, attrs_to_del=attrs_to_del, var_to_del=var_to_del,
pass_col_name=pass_col, attrs=attrs, attrs_to_del=attrs_to_del, var_to_del=var_to_del,
filename_format=filename_format,
is_full_time=full_time, extract_size_km=pass_width * 2, var_attr_edit=var_attr_edit,
specific_func=specfic_func)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment