Commit 65f2fe8f authored by ARCHER's avatar ARCHER 💬
Browse files

change wind_to_direction to wind_from_direction

also skip fillvalue is none is defined upstream
parent dea91753
...@@ -27,7 +27,7 @@ def readFile(fname): ...@@ -27,7 +27,7 @@ def readFile(fname):
try: try:
nc=netCDF4.Dataset(fname) nc=netCDF4.Dataset(fname)
except Exception as e: except Exception as e:
logger.warn('skipping unreadable %s : %s' % (fname, str(e))) logger.warning('skipping unreadable %s : %s' % (fname, str(e)))
return None return None
# read owiLon to get valid shape # read owiLon to get valid shape
...@@ -71,11 +71,11 @@ def readFile(fname): ...@@ -71,11 +71,11 @@ def readFile(fname):
def readMeta(fname): def readMeta(fname):
data={} data={}
#list_var=('owiLon','owiLat','owiHeading','owiEcmwfWindSpeed','owiWindSpeed_IPF', 'owiPreProcessing/estimWindDir200', 'owiPreProcessing/MixWindDir200', 'owiPreProcessing/CoefMixWindDir200', 'owiEcmwfWindDirection','owiWindDirection_mod1','owiWindSpeed_mod1','owiWindDirection_mod2','owiWindSpeed_mod2','owiWindDirection_mod3','owiWindSpeed_mod3','owiPreProcessing/FilterBinary200','owiLandFlag','owiIncidenceAngle') #list_var=('owiLon','owiLat','owiHeading','owiEcmwfWindSpeed','owiWindSpeed_IPF', 'owiPreProcessing/estimWindDir200', 'owiPreProcessing/MixWindDir200', 'owiPreProcessing/CoefMixWindDir200', 'owiEcmwfWindDirection','owiWindDirection_mod1','owiWindSpeed_mod1','owiWindDirection_mod2','owiWindSpeed_mod2','owiWindDirection_mod3','owiWindSpeed_mod3','owiPreProcessing/FilterBinary200','owiLandFlag','owiIncidenceAngle')
attrs = ['long_name','units','valid_range','flag_values','flag_meanings'] attrs = ['long_name', 'units', 'valid_range', 'flag_values','flag_meanings', '_FillValue']
try: try:
nc=netCDF4.Dataset(fname) nc=netCDF4.Dataset(fname)
except Exception as e: except Exception as e:
logger.warn('skipping unreadable %s : %s' % (fname, str(e))) logger.warning('skipping unreadable %s : %s' % (fname, str(e)))
return None return None
# read globals attr # read globals attr
...@@ -148,7 +148,7 @@ def writeLight(fname,data,date,meta=None): ...@@ -148,7 +148,7 @@ def writeLight(fname,data,date,meta=None):
('owiLon' , 'lon'), ('owiLon' , 'lon'),
('owiLat' , 'lat'), ('owiLat' , 'lat'),
('owiWindSpeed' , 'wind_speed'), ('owiWindSpeed' , 'wind_speed'),
('owiWindDirection' , 'wind_to_direction'), ('owiWindDirection' , 'wind_from_direction'),
('owiPreProcessing/wind_streaks_orientation' , 'wind_streaks_orientation'), ('owiPreProcessing/wind_streaks_orientation' , 'wind_streaks_orientation'),
('owiPreProcessing/wind_streaks_orientation_stddev' , 'wind_streaks_orientation_stddev'), ('owiPreProcessing/wind_streaks_orientation_stddev' , 'wind_streaks_orientation_stddev'),
# ('owiWindQuality' , 'quality_flag'), # ('owiWindQuality' , 'quality_flag'),
...@@ -191,8 +191,8 @@ def writeLight(fname,data,date,meta=None): ...@@ -191,8 +191,8 @@ def writeLight(fname,data,date,meta=None):
'standard_name' : 'latitude', 'standard_name' : 'latitude',
'units' : 'degrees_north' 'units' : 'degrees_north'
}, },
'wind_to_direction': { 'wind_from_direction': {
'long_name' : 'Wind to direction (oceanographic convention)', 'long_name' : 'Wind to direction (meteorologic convention)',
'units' : 'degrees', 'units' : 'degrees',
# 'usage_level' : "basic" # 'usage_level' : "basic"
} }
...@@ -236,13 +236,16 @@ def writeLight(fname,data,date,meta=None): ...@@ -236,13 +236,16 @@ def writeLight(fname,data,date,meta=None):
if in_name in data: if in_name in data:
value = data[in_name].T value = data[in_name].T
try: try:
fill_value=data[in_name].fill_value fill_value=meta[in_name]['_FillValue']
if fill_value is None:
fill_value=False
except: except:
fill_value=None fill_value = data[in_name].fill_value
# transpose dims for correct netcdf ordering # transpose dims for correct netcdf ordering
dtype = value.dtype dtype = value.dtype
if dtype in translate_dtype: if dtype in translate_dtype:
dtype = translate_dtype[dtype] dtype = translate_dtype[dtype]
logger.warning('%s fill_value: %s' % (out_name, fill_value))
ncvar = nc.createVariable(out_name,dtype,dimensions=('time','y','x'),fill_value=fill_value) ncvar = nc.createVariable(out_name,dtype,dimensions=('time','y','x'),fill_value=fill_value)
for _attr in attrs: for _attr in attrs:
...@@ -391,12 +394,16 @@ def nclight2gridded(filein,fileout,crs_out=None): ...@@ -391,12 +394,16 @@ def nclight2gridded(filein,fileout,crs_out=None):
for var in list((set(ds_sw.coords.keys()) | set(ds_sw.keys())).difference(set(['time']))): for var in list((set(ds_sw.coords.keys()) | set(ds_sw.keys())).difference(set(['time']))):
# set up dataarray. dtype is forced to np.float64, because we use Nan for interpolation # set up dataarray. dtype is forced to np.float64, because we use Nan for interpolation
#ds_sw[var].values.dtype #ds_sw[var].values.dtype
da_var = xr.DataArray(np.full(da_lon.shape, ds_sw[var]._FillValue, dtype=np.float64),coords=[ds_sw.time,y, x], dims=['time','y', 'x']) try:
fillvalue=ds_sw[var]._FillValue
except:
fillvalue=netCDF4.default_fillvals['f8']
da_var = xr.DataArray(np.full(da_lon.shape, fillvalue, dtype=np.float64),coords=[ds_sw.time,y, x], dims=['time','y', 'x'])
# fill dataarray # fill dataarray
da_var.values[0,y_idx,x_idx] = ds_sw[var].values da_var.values[0,y_idx,x_idx] = ds_sw[var].values
# save nan, as they are going to be interpolated # save nan, as they are going to be interpolated
nan_mask = np.isnan(da_var.values) nan_mask = np.isnan(da_var.values)
da_var.values[nan_mask] = ds_sw[var]._FillValue # not really needed da_var.values[nan_mask] = fillvalue # not really needed
try: try:
da_var.values[0,y_interp,x_interp] = np.nan da_var.values[0,y_interp,x_interp] = np.nan
da_var = da_var.ffill(dim='x',limit=2) da_var = da_var.ffill(dim='x',limit=2)
...@@ -410,25 +417,30 @@ def nclight2gridded(filein,fileout,crs_out=None): ...@@ -410,25 +417,30 @@ def nclight2gridded(filein,fileout,crs_out=None):
da_var.attrs = ds_sw[var].attrs da_var.attrs = ds_sw[var].attrs
da_var.rio.set_spatial_dims('x','y',inplace=True) da_var.rio.set_spatial_dims('x','y',inplace=True)
ds_gd[var] = da_var ds_gd[var] = da_var
encoding={}
for var in list(set(ds_gd.coords.keys()) | set(ds_gd.keys())): for var in list(set(ds_gd.coords.keys()) | set(ds_gd.keys())):
encoding[var] = {}
if var in meta_over: if var in meta_over:
# python 3 only (merge dict ) # python 3 only (merge dict )
# ds_gd[var].attrs = { **ds_sw[var].attrs,**meta_over[var] } # ds_gd[var].attrs = { **ds_sw[var].attrs,**meta_over[var] }
# python 2 compat: # python 2 compat:
ds_gd[var].attrs.update(meta_over[var]) ds_gd[var].attrs.update(meta_over[var])
if '_FillValue' not in ds_gd[var].attrs:
# disable automatic fillvalue in fileout
encoding[var].update({'_FillValue': None})
ds_gd.rio.set_spatial_dims('x','y', inplace=True) ds_gd.rio.set_spatial_dims('x','y', inplace=True)
ds_gd.rio.write_crs(crs, inplace=True) ds_gd.rio.write_crs(crs, inplace=True)
encoding=None #encoding={'lat': {'_FillValue': None}, 'lon': {'_FillValue': None}}
# final reprojection on crs_out # final reprojection on crs_out
if crs_out is not None: if crs_out is not None:
logger.info("reprojecting to final proj %s" % str(crs_out)) logger.info("reprojecting to final proj %s" % str(crs_out))
ds_gd = ds_gd.rio.reproject(crs_out) ds_gd = ds_gd.rio.reproject(crs_out)
if pyproj.CRS(crs_out).is_geographic : if pyproj.CRS(crs_out).is_geographic :
# crs out is lon/lat : rename x,y to lon,lat # crs out is lon/lat : rename x,y to lon,lat
ds_gd = ds_gd.drop(['lon','lat']).rename({'x':'lon','y':'lat'}) ds_gd = ds_gd.drop(['lon','lat']).rename({'x':'lon','y':'lat'})
ds_gd['lon'].attrs = { ds_gd['lon'].attrs = {
...@@ -441,7 +453,8 @@ def nclight2gridded(filein,fileout,crs_out=None): ...@@ -441,7 +453,8 @@ def nclight2gridded(filein,fileout,crs_out=None):
"standard_name" : "latitude", "standard_name" : "latitude",
"units" : "degrees_north" "units" : "degrees_north"
} }
encoding = {'lat': {'_FillValue': None}, 'lon': {'_FillValue': None}} del encoding['x']
del encoding['y']
ds_gd.rio.set_spatial_dims('lon','lat', inplace=True) ds_gd.rio.set_spatial_dims('lon','lat', inplace=True)
ds_gd.rio.write_crs(crs_out, inplace=True) ds_gd.rio.write_crs(crs_out, inplace=True)
...@@ -451,7 +464,7 @@ def getAttribute(fname, attr): ...@@ -451,7 +464,7 @@ def getAttribute(fname, attr):
try: try:
nc=netCDF4.Dataset(fname) nc=netCDF4.Dataset(fname)
except Exception as e: except Exception as e:
logger.warn('skipping unreadable %s : %s' % (fname, str(e))) logger.warning('skipping unreadable %s : %s' % (fname, str(e)))
return None return None
if attr in nc.ncattrs(): if attr in nc.ncattrs():
...@@ -462,7 +475,7 @@ def getFootprint(fname): ...@@ -462,7 +475,7 @@ def getFootprint(fname):
try: try:
nc=netCDF4.Dataset(fname) nc=netCDF4.Dataset(fname)
except Exception as e: except Exception as e:
logger.warn('skipping unreadable %s : %s' % (fname, str(e))) logger.warning('skipping unreadable %s : %s' % (fname, str(e)))
return None return None
if "footprint" in nc.ncattrs(): if "footprint" in nc.ncattrs():
...@@ -498,7 +511,7 @@ def getDataAtCoord(data,lon,lat): ...@@ -498,7 +511,7 @@ def getDataAtCoord(data,lon,lat):
mindist=np.min(dist) mindist=np.min(dist)
imindist=np.argmin(dist) imindist=np.argmin(dist)
if mindist > 20: # if mindist > 20: #
logger.warn("Buoy to far from sar image (%s km)" % mindist) logger.warning("Buoy to far from sar image (%s km)" % mindist)
return None return None
data['dist_km']=mindist data['dist_km']=mindist
......
...@@ -12,6 +12,6 @@ setup(name='owi', ...@@ -12,6 +12,6 @@ setup(name='owi',
zip_safe=False, zip_safe=False,
scripts=['bin/owi2nclight.py','bin/nclight2gridded.py'], scripts=['bin/owi2nclight.py','bin/nclight2gridded.py'],
install_requires=[ install_requires=[
'numpy', 'netCDF4' 'numpy', 'netCDF4', 'future'
]) ])
#scripts=['bin/pathUrl.py']) #scripts=['bin/pathUrl.py'])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment