New attributes for SMAP and SMOS

bin/extractSmapCyclone.py

@@ -57,20 +57,22 @@ if __name__ == "__main__":
     attrs = {"Conventions": "CF-1.6",
              "title": "SMAP ocean surface wind speed (10m above surface) cropped around Tropical Cyclone track",
              "institution": "IFREMER/LOPS",
-             "sourceProduct": "",
-             "sourceReference": """Meissner, T., L. Ricciardulli, and 
-                 F. Wentz, 2018: Remote Sensing Systems SMAP daily Sea Surface Winds 
-                Speeds on 0.25 deg grid, Version 01.0. [NRT or FINAL]. Remote Sensing 
-                Systems, Santa Rosa, CA. Available online at 
-                www.remss.com/missions/smap/""",
-             "missionName": "SMAP"
+             "reference": "https://cyclobs.ifremer.fr/app/about#smap_ref",
+             "sourceReference": ("Meissner, T., L. Ricciardulli, and F. Wentz, 2018: Remote Sensing Systems SMAP daily"
+                                 "Sea Surface Winds Speeds on 0.25 deg grid, Version 01.0. [NRT or FINAL]. Remote "
+                                 "Sensing Systems, Santa Rosa, CA. Available online at www.remss.com/missions/smap/"),
+             "missionName": "SMAP",
+             "wind": {"_FillValue": -9999}
              }
 
+    attrs_to_del = {"wind": ["missing_value"]}
+
     for f in args.input:
         # TODO generate filename_format depending on input filename
         process_file(Session(), file=f, output_path=args.output, extract_date_func=extract_date_from_filename,
                      var_to_del=["minute"],
                      attrs=attrs,
+                     attrs_to_del=attrs_to_del,
                      wind_col="wind", time_col="minute",
                      lat_col="lat",
                      lon_col="lon", pass_col="node", pass_width=1000,

bin/extractSmosCyclone.py

@@ -40,7 +40,7 @@ def set_nan_low_quality(dataset):
 
     quality_col = "quality_level"
     # set nan to quality levels > 1 because
-    with np.errstate(invalid='ignore'): # ignoring warnings caused by Nans in array
+    with np.errstate(invalid='ignore'):  # ignoring warnings caused by Nans in array
         dataset[quality_col] = dataset[quality_col].where(dataset[quality_col].data <= 1)
 
     # Sets equivalents NaNs to other variables
@@ -81,26 +81,27 @@ if __name__ == "__main__":
     Session = sessionmaker(bind=engine)
 
     # Attributes that'll be deleted
-    #attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
+    # attrs_to_del = ["aprrox_local_equatorial_crossing_time", "swath_sector", "geospatial_bounds",
     #                "geospatial_lat_min", "geospatial_lat_max", "geospatial_lon_min", "geospatial_lon_max"]
 
     attrs = {"Conventions": "CF-1.6",
              "title": "SMOS ocean surface wind speed cropped around Tropical Cyclone track",
              "institution": "IFREMER/LOPS",
-             "sourceProduct": "",
-             "sourceReference": """Reul Nicolas, Tenerelli Joseph, Chapron Bertrand, Vandemark Doug, Quilfen
-              Yves, Kerr Yann (2012). SMOS satellite L-band radiometer: A new capability for ocean surface
-              remote sensing in hurricanes. Journal Of Geophysical Research-oceans, 117. Publisher's official version : 
-              http://dx.doi.org/10.1029/2011JC007474""",
+             "reference": "https://cyclobs.ifremer.fr/app/about#smos_ref",
+             "sourceReference": ("Reul Nicolas, Tenerelli Joseph, Chapron Bertrand, Vandemark Doug, Quilfen "
+                                 "Yves, Kerr Yann (2012). SMOS satellite L-band radiometer: A new capability for "
+                                 "ocean surface remote sensing in hurricanes. Journal Of Geophysical Research-oceans, "
+                                 "117. Publisher's official version :  http://dx.doi.org/10.1029/2011JC007474"),
              "missionName": "SMOS"
              }
+    attrs_to_del = {"time": ["authority"], "lat": ["authority"], "lon": ["authority"], "wind_speed": ["authority"]}
 
     from extract_cyclones import process_file
 
     for f in args.input:
         # TODO generate filename_format depending on input filename
         process_file(Session(), file=f, output_path=args.output, extract_date_func=extract_date_from_filename,
-                     var_to_del=["measurement_time"], wind_col="wind_speed",
+                     var_to_del=["measurement_time"], wind_col="wind_speed", attrs_to_del=attrs_to_del,
                      attrs=attrs,
                      time_col="measurement_time",
                      lat_col="lat", lon_col="lon", pass_col=None, pass_width=1200,

extract_cyclones/__init__.py

@@ -183,7 +183,7 @@ def extract_start_stop_measure(dataset, time_col_name):
 
 
 def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path,
-                               file_date, attrs, attrs_to_del, var_to_del, lon_col_name,
+                               file_date, source_filename, attrs, attrs_to_del, var_to_del, lon_col_name,
                                lat_col_name, wind_col_name, time_col_name, pass_col_name,
                                filename_format,
                                is_full_time,
@@ -287,13 +287,27 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
 
         # Setting new attributes
         for attr in attrs:
-            sel.attrs[attr] = attrs[attr]
+            if type(attrs[attr]) == dict:
+                for var_attr in attrs[attr]:
+                    if var_attr == "_FillValue":
+                        sel[attr].encoding[var_attr] = attrs[attr][var_attr]
+                    else:
+                        sel[attr].attrs[var_attr] = attrs[attr][var_attr]
+            else:
+                sel.attrs[attr] = attrs[attr]
 
         # Removing attributes
         # FIXME potentially to remove because shadowed by the above deletion
         for attr in attrs_to_del:
-            if attr in sel.attrs:
-                del sel.attrs[attr]
+            if type(attrs_to_del[attr]) == list:
+                for var_attr in attrs_to_del[attr]:
+                    if var_attr in sel[attr].attrs:
+                        del sel[attr].attrs[var_attr]
+                    if var_attr in sel[attr].encoding:
+                        del sel[attr].encoding[var_attr]
+            else:
+                if attr in sel.attrs:
+                    del sel.attrs[attr]
 
         for var in var_to_del:
             if var in sel.keys():
@@ -309,6 +323,7 @@ def extract_write_cyclone_data(dataset, kept_track_points, filename, output_path
         sel[lon_col_name] = deg180(sel[lon_col_name])
         sel[lon_col_name].attrs = lon_attrs
 
+        sel.attrs["sourceProduct"] = source_filename
         sel.attrs["footprint"] = dumps(footprint_polyg)
 
         for var in var_attr_edit:
@@ -358,7 +373,8 @@ def process_file(session, file, output_path, extract_date_func, attrs, var_to_de
                                                        pass_col_name=pass_col, is_full_time=full_time)
         logger.debug(f"For file {filename} Kept track that will be used to extract cyclone data: {kept_track_points}")
 
-        extract_write_cyclone_data(dataset, kept_track_points, filename, output_path, file_date, wind_col_name=wind_col,
+        extract_write_cyclone_data(dataset, kept_track_points, filename, output_path, file_date,
+                                   source_filename=filename, wind_col_name=wind_col,
                                    time_col_name=time_col, lat_col_name=lat_col, lon_col_name=lon_col,
                                    pass_col_name=pass_col, attrs=attrs, attrs_to_del=attrs_to_del, var_to_del=var_to_del,
                                    filename_format=filename_format,