Modified SMOS, SMAP attributes. Not writing files that have below 5 pixels.

bin/extractSmapCyclone.py

@@ -50,14 +50,27 @@ if __name__ == "__main__":
     engine = create_engine(args.dbd, pool_size=50, max_overflow=0)
     Session = sessionmaker(bind=engine)
 
-    attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "first_orbit", "last_orbit"]
+    #attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "first_orbit", "last_orbit"]
 
-    var_attr_edit = {"lon": {"valid_min": -180, "valid_max": 180}}
+    #var_attr_edit = {"lon": {"valid_min": -180, "valid_max": 180}}
+
+    attrs = {"Conventions": "CF-1.6",
+             "title": "SMAP ocean surface wind speed (10m above surface) cropped around Tropical Cyclone track",
+             "institution": "IFREMER/LOPS",
+             "sourceProduct": "",
+             "sourceReference": """Meissner, T., L. Ricciardulli, and 
+                 F. Wentz, 2018: Remote Sensing Systems SMAP daily Sea Surface Winds 
+                Speeds on 0.25 deg grid, Version 01.0. [NRT or FINAL]. Remote Sensing 
+                Systems, Santa Rosa, CA. Available online at 
+                www.remss.com/missions/smap/""",
+             "missionName": "SMAP"
+             }
 
     for f in args.input:
         # TODO generate filename_format depending on input filename
         process_file(Session(), file=f, output_path=args.output, extract_date_func=extract_date_from_filename,
-                     attrs_to_del=attrs_to_del, var_attr_edit=var_attr_edit, var_to_del=["minute"],
+                     var_to_del=["minute"],
+                     attrs=attrs,
                      wind_col="wind", time_col="minute",
                      lat_col="lat",
                      lon_col="lon", pass_col="node", pass_width=1000,

bin/extractSmosCyclone.py

@@ -81,15 +81,27 @@ if __name__ == "__main__":
     Session = sessionmaker(bind=engine)
 
     # Attributes that'll be deleted
-    attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
-                    "geospatial_lat_min", "geospatial_lat_max", "geospatial_lon_min", "geospatial_lon_max"]
+    #attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
+    #                "geospatial_lat_min", "geospatial_lat_max", "geospatial_lon_min", "geospatial_lon_max"]
+
+    attrs = {"Conventions": "CF-1.6",
+             "title": "SMOS ocean surface wind speed cropped around Tropical Cyclone track",
+             "institution": "IFREMER/LOPS",
+             "sourceProduct": "",
+             "sourceReference": """Reul Nicolas, Tenerelli Joseph, Chapron Bertrand, Vandemark Doug, Quilfen
+              Yves, Kerr Yann (2012). SMOS satellite L-band radiometer: A new capability for ocean surface
+              remote sensing in hurricanes. Journal Of Geophysical Research-oceans, 117. Publisher's official version : 
+              http://dx.doi.org/10.1029/2011JC007474""",
+             "missionName": "SMOS"
+             }
 
     from extract_cyclones import process_file
 
     for f in args.input:
         # TODO generate filename_format depending on input filename
         process_file(Session(), file=f, output_path=args.output, extract_date_func=extract_date_from_filename,
-                     attrs_to_del=attrs_to_del, var_to_del=["measurement_time"], wind_col="wind_speed",
+                     var_to_del=["measurement_time"], wind_col="wind_speed",
+                     attrs=attrs,
                      time_col="measurement_time",
                      lat_col="lat", lon_col="lon", pass_col=None, pass_width=1200,
                      filename_format="SM_OPER_MIR_<time>_<sid>.nc", specfic_func=set_nan_low_quality)

extract_cyclones/__init__.py

@@ -164,7 +164,6 @@ def compute_footprint(dataset, wind_field):
     df = dataset.to_dataframe()
     df = df[pd.isna(df[wind_field]) == False]
     df = df.reset_index()
-
     gdf = gpd.GeoDataFrame(
         df, geometry=gpd.points_from_xy(df.lon, df.lat))
 
@@ -184,13 +183,14 @@ def extract_start_stop_measure(dataset, time_col_name):
 
 
 def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path,
-                               file_date, attrs_to_del, var_to_del, lon_col_name,
+                               file_date, attrs, attrs_to_del, var_to_del, lon_col_name,
                                lat_col_name, wind_col_name, time_col_name, pass_col_name,
                                filename_format,
                                is_full_time,
                                extract_size_km=2000,
                                var_attr_edit={},
-                               specific_func=None):
+                               specific_func=None,
+                               min_pixel=5):
     for sid, track_point in kept_track_points.items():
         # Km per deg in latitude
         km_per_deg_lat = 111
@@ -253,6 +253,13 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
         if specific_func is not None:
             sel = specific_func(sel)
 
+        # if there are less than x points, remove the image
+        df = sel.to_dataframe()
+        df = df[pd.isna(df[wind_col_name]) == False]
+        if len(df.index) <= min_pixel:
+            logger.info(f"Data has less than {min_pixel} pixel of data. Not writing to file.")
+            return
+
         # Extracting min and max time to set attributes on NetCDF
         min_time, max_time = extract_start_stop_measure(sel, time_col_name)
         if max_time - min_time > datetime.timedelta(minutes=30):
@@ -261,18 +268,29 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
 
         logger.debug(f"min_time: {min_time}, max_time: {max_time}")
         if is_full_time:
-            sel.attrs["measurement_start_date"] = min_time.strftime("%Y-%m-%d %H:%M:%S")
-            sel.attrs["measurement_stop_date"] = max_time.strftime("%Y-%m-%d %H:%M:%S")
+            #sel.attrs["measurement_start_date"] = min_time.strftime("%Y-%m-%d %H:%M:%S")
+            #sel.attrs["measurement_stop_date"] = max_time.strftime("%Y-%m-%d %H:%M:%S")
+            sel.attrs["measurementDate"] = (min_time + (max_time - min_time) / 2).strftime("%Y-%m-%d %H:%M:%S")
             year = str(min_time.year)
             doy = min_time.strftime('%j')
         else:
             min_date = file_date + min_time
+            max_date = file_date + max_time
             year = min_date.year
             doy = min_date.strftime('%j')
-            sel.attrs["measurement_start_date"] = str(min_date)
-            sel.attrs["measurement_stop_date"] = str(file_date + max_time)
+            #sel.attrs["measurement_start_date"] = str(min_date)
+            #sel.attrs["measurement_stop_date"] = str(file_date + max_time)
+            sel.attrs["measurementDate"] = (min_date + (max_date - min_date) / 2).strftime("%Y-%m-%d %H:%M:%S")
+
+        # Deleting all attributes
+        sel.attrs.clear()
+
+        # Setting new attributes
+        for attr in attrs:
+            sel.attrs[attr] = attrs[attr]
 
         # Removing attributes
+        # FIXME potentially to remove because shadowed by the above deletion
         for attr in attrs_to_del:
             if attr in sel.attrs:
                 del sel.attrs[attr]
@@ -313,8 +331,9 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
         sel.to_netcdf(os.path.join(out, output_filename), format="NETCDF4_CLASSIC")
 
 
-def process_file(session, file, output_path, extract_date_func, attrs_to_del, var_to_del, wind_col,
-                 time_col, lat_col, lon_col, filename_format, pass_width, pass_col=None, var_attr_edit={},
+def process_file(session, file, output_path, extract_date_func, attrs, var_to_del, wind_col,
+                 time_col, lat_col, lon_col, filename_format, pass_width, pass_col=None, attrs_to_del=[],
+                 var_attr_edit={},
                  specfic_func=None):
     logger.info(f"Processing {file}...")
 
@@ -341,7 +360,7 @@ def process_file(session, file, output_path, extract_date_func, attrs_to_del, va
 
         extract_write_cyclone_data(dataset, kept_track_points, filename, output_path, file_date, wind_col_name=wind_col,
                                    time_col_name=time_col, lat_col_name=lat_col, lon_col_name=lon_col,
-                                   pass_col_name=pass_col, attrs_to_del=attrs_to_del, var_to_del=var_to_del,
+                                   pass_col_name=pass_col, attrs=attrs, attrs_to_del=attrs_to_del, var_to_del=var_to_del,
                                    filename_format=filename_format,
                                    is_full_time=full_time, extract_size_km=pass_width * 2, var_attr_edit=var_attr_edit,
                                    specific_func=specfic_func)