Commit d306f3d8 authored by Cyrille Poncelet's avatar Cyrille Poncelet
Browse files

remove ulong references, and update python code generator to handle user

define types at group level
parent e63d1acf
/.metadata/
......@@ -149,7 +149,7 @@
"dims": [
"profile"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -445,7 +445,7 @@
"dims": [
"profile"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -711,7 +711,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -764,7 +764,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -854,7 +854,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -936,7 +936,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -991,7 +991,7 @@
"dims": [
"clock_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -1030,7 +1030,7 @@
"dims": [
"clock_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -1166,7 +1166,7 @@
"dims": [
"height_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -1364,7 +1364,7 @@
"dims": [
"depth_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -1570,7 +1570,7 @@
"dims": [
"heading_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -1648,7 +1648,7 @@
"dims": [
"heading_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......
......@@ -1473,7 +1473,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -1936,7 +1936,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -2014,7 +2014,7 @@
"dims": [
"time"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -2238,7 +2238,7 @@
"dims": [
"runtime_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -2925,7 +2925,7 @@
"dims": [
"extra_parameters_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -3044,7 +3044,7 @@
"dims": [
"dg_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......
......@@ -22,10 +22,6 @@
"name": "float(*) sample_t",
"comment": "Variable length vector used to store ragged arrays of backscatter data. Data type can be varied to suit data storage needs."
},
{
"name": "byte(*) sample_byte_t",
"comment": "Variable length vector used to store ragged arrays of backscatter data. Data type can be varied to suit data storage needs."
},
{
"name": "float(*) angle_t",
"comment": "Variable length vector used to store ragged arrays of split-aperture angles. Data type can varied to suit data storage needs."
......@@ -154,7 +150,7 @@
"beam",
"subbeam"
],
"type": "sample_byte_t",
"type": "sample_t",
"attributes": [
{
"name": "long_name",
......@@ -3675,7 +3671,7 @@
"dims": [
"beamdescription"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "calendar",
......@@ -4357,7 +4353,7 @@
"ping_time",
"rx_antenna_count"
],
"type": "ulong",
"type": "uint64",
"attributes": [
{
"name": "long_name",
......
......@@ -37,7 +37,7 @@ public class PythonSourceCodeGenerator {
if (!outputDir.exists())
Files.createDirectories(outputDir.toPath());
Path outputFolder=outputDir.toPath();
Files.write(outputFolder.resolve(Paths.get("xsf_constants.py")),
Files.write(outputFolder.resolve(Paths.get("sonar_groups.py")),
SCGroupAdapter.getPythonSourceCode(group,"","",false));
}
......
......@@ -239,7 +239,8 @@ public class SCGroupAdapter {
case "string":
value="\""+att.value+"\"";
String comment = att.comment ==null ? "" : "#"+att.comment;
sourceCode.add(String.format("%s%s%s.%s = %s #%s", localIdent, pythonIndent,grouporVariableName,att.name,value,comment));
sourceCode.add(String.format("%s%sif \"%s\" not in attributes:", localIdent, pythonIndent,att.name));
sourceCode.add(String.format("%s%s%s%s.%s = %s #%s", localIdent, pythonIndent,pythonIndent,grouporVariableName,att.name,value,comment));
break;
case "float":
value=att.value.replace("f", "");
......@@ -248,7 +249,8 @@ public class SCGroupAdapter {
value="np.float32("+value+")";
comment = att.comment ==null ? "" : " #"+att.comment;
sourceCode.add(String.format("%s%s%s.%s = %s%s", localIdent, pythonIndent,grouporVariableName,att.name,value,comment));
sourceCode.add(String.format("%s%sif \"%s\" not in attributes:", localIdent, pythonIndent,att.name));
sourceCode.add(String.format("%s%s%s%s.%s = %s%s", localIdent, pythonIndent,pythonIndent,grouporVariableName,att.name,value,comment));
break;
case "int":
value=att.value;
......@@ -260,19 +262,22 @@ public class SCGroupAdapter {
value="["+value+"]";
comment = att.comment ==null ? "" : "#"+att.comment;
sourceCode.add(String.format("%s%s%s.%s = %s%s", localIdent, pythonIndent,grouporVariableName,att.name.trim(),value,comment));
sourceCode.add(String.format("%s%sif \"%s\" not in attributes:", localIdent, pythonIndent,att.name));
sourceCode.add(String.format("%s%s%s%s.%s = %s%s", localIdent,pythonIndent, pythonIndent,grouporVariableName,att.name.trim(),value,comment));
break;
default:
value=att.value;
comment = att.comment ==null ? "" : "#"+att.comment;
sourceCode.add(String.format("%s%s#%s.%s = %s%s", localIdent, pythonIndent,grouporVariableName,att.name,value,comment));
sourceCode.add(String.format("%s%s#if \"%s\" not in attributes:", localIdent, pythonIndent,att.name));
sourceCode.add(String.format("%s%s%s#%s.%s = %s%s", localIdent, pythonIndent,pythonIndent,grouporVariableName,att.name,value,comment));
}
} else
{
value="\""+att.value+"\"";
String comment = att.comment ==null ? "" : "#"+att.comment;
sourceCode.add(String.format("%s%s%s.%s = %s%s", localIdent, pythonIndent,grouporVariableName,att.name,value,comment));
sourceCode.add(String.format("%s%sif \"%s\" not in attributes:", localIdent, pythonIndent,att.name));
sourceCode.add(String.format("%s%s%s%s.%s = %s%s", localIdent, pythonIndent,pythonIndent,grouporVariableName,att.name,value,comment));
}
}
......@@ -294,14 +299,12 @@ public class SCGroupAdapter {
"https://gitlab.ifremer.fr/fleet/XSF"));
sourceCode.add("\nimport netCDF4 as nc");
sourceCode.add("\nimport numpy as np");
sourceCode.add("\nfrom .nc_tools import find_type");
sourceCode.add("# pylint: disable=E1101");
sourceCode.add("# pylint: disable=R0904");
sourceCode.add("# pylint: disable=C0305");
sourceCode.add("# pylint: disable=C0302");
sourceCode.add("# global container for type");
sourceCode.add("types = dict()");
}
String className=getClassName(group, "");
if (group.groupCategory == GroupCategory.eSensor) {
......@@ -334,22 +337,12 @@ public class SCGroupAdapter {
sourceCode.add(String.format("%sdef get_group_path():\n%s%sreturn %s", localIdent,localIdent,pythonIndent, String.format("%s+\"%s/\"", pathGetter,group.name)));
}
}
if(is_sensor_associated)
{
sourceCode.add(String.format("\n%sdef create_group(self, parent_group: nc.Dataset, ident):", localIdent));
sourceCode.add(String.format("%s%sg = parent_group.createGroup(ident)", localIdent, pythonIndent));
}
else
{
sourceCode.add(String.format("\n%sdef create_group(self, parent_group: nc.Dataset):", localIdent));
sourceCode.add(String.format("%s%sg = parent_group.createGroup(self.get_group_path())", localIdent, pythonIndent));
}
List<String> enumDeclaration = new ArrayList<>() ;
List<String> vlenTypeDict = new ArrayList<>() ;
//parse groupe type declaration
for(Type type:group.getTypes())
{
sourceCode.add(String.format("%s%s#type %s %s %s", localIdent,pythonIndent,type.getName(),type.getComment(),type.getDescription()));
if(type.getName().contains("enum"))
{
String data[]=type.getName().split(" enum ");
......@@ -369,9 +362,9 @@ public class SCGroupAdapter {
dictionary+="'"+enums[0].trim()+"': "+enums[1].trim().replace("\u00A0", "")+", ";
}
dictionary+="}";
sourceCode.add(String.format("%s%s%s", localIdent, pythonIndent,dictionary));
sourceCode.add(String.format("%s%stype_e = g.createEnumType(np.uint8, '%s', enum_dict)", localIdent, pythonIndent,name));
sourceCode.add(String.format("%s%stypes['%s'] = type_e", localIdent, pythonIndent,name));
enumDeclaration.add(String.format("%s%s#type %s %s %s", localIdent,pythonIndent,type.getName(),type.getComment(),type.getDescription()));
enumDeclaration.add(String.format("%s%s%s", localIdent, pythonIndent,dictionary));
enumDeclaration.add(String.format("%s%stype_e = g.createEnumType(np.uint8, '%s', enum_dict)", localIdent, pythonIndent,name));
}else if (type.getName().contains("*"))
{
//this is a vlen type
......@@ -400,23 +393,61 @@ public class SCGroupAdapter {
default:
throw new NotImplementedException("Not supported vlen type");
}
sourceCode.add(String.format("%s%svlen_t = g.createVLType(%s, \"%s\")", localIdent, pythonIndent,nptype,name));
sourceCode.add(String.format("%s%stypes['%s'] = vlen_t", localIdent, pythonIndent,name));
//create a dictionnary with python types
String comment = String.format("%s%s#type %s %s %s",localIdent," ",type.getName(),type.getComment(),type.getDescription());
vlenTypeDict.add(String.format("%s\n%s%s\"%s\":%s",comment,localIdent," ",name,nptype));
}
}
String vlenTypeDictDeclaration= "vlen_type_dict={}";
if(!vlenTypeDict.isEmpty())
{
vlenTypeDictDeclaration=String.format("\n%s%svlen_type_dict={\n%s}",localIdent," ", String.join(",\n",vlenTypeDict));
}
if(is_sensor_associated)
{
sourceCode.add(String.format("\n%sdef create_group(self, parent_group: nc.Dataset, ident,%s,**attributes):", localIdent,vlenTypeDictDeclaration));
sourceCode.add(String.format("%s%sg = parent_group.createGroup(ident)", localIdent, pythonIndent));
}
else
{
sourceCode.add(String.format("\n%sdef create_group(self, parent_group: nc.Dataset,%s,**attributes):", localIdent,vlenTypeDictDeclaration));
sourceCode.add(String.format("%s%sg = parent_group.createGroup(self.get_group_path())", localIdent, pythonIndent));
}
//declare enum
if(!enumDeclaration.isEmpty())
{
sourceCode.addAll(enumDeclaration);
}
//Vlen types
{
sourceCode.add(String.format("\n%s%s#declare vlen types:", localIdent,pythonIndent));
sourceCode.add(String.format("%s%sfor key, value in vlen_type_dict.items():", localIdent,pythonIndent));
sourceCode.add(String.format("%s%s%svlen_type = g.createVLType(vlen_type_dict[key], key)", localIdent,pythonIndent,pythonIndent));
}
//Attributes
for(Attribute att : group.attributes)
{
writeAttribute(sourceCode,att,localIdent,"g");
}
sourceCode.add(String.format("%s%sreturn g", localIdent, pythonIndent));
sourceCode.add("\n"+localIdent+ pythonIndent + String.format("#write attributes passed as argument"));
sourceCode.add(localIdent +pythonIndent + String.format("for key, value in attributes.items():"));
sourceCode.add(localIdent + pythonIndent+pythonIndent + String.format("g.setncattr(key, value)"));
sourceCode.add(String.format("\n%s%sreturn g", localIdent, pythonIndent));
// Dimension name code generation
if (!group.dims.isEmpty()) {
sourceCode.add(localIdent + "# dimensions names declaration");
sourceCode.add("\n"+localIdent + "# dimensions names declaration");
group.dims.forEach(d -> sourceCode.add(localIdent + String.format("%s = \"%s\"",
SCDimensionAdapter.getDimNameDeclaration(d), SCDimensionAdapter.getDimNCName(d))));
......@@ -476,6 +507,7 @@ public class SCGroupAdapter {
boolean has_fill_value=true;
String fill_value="np.nan";
has_fill_value=false; //set to false by default
boolean is_vlen_or_conpound=false;
switch(v.getType())
{
......@@ -519,8 +551,9 @@ public class SCGroupAdapter {
datatype="np.ubyte";
break;
default:
datatype="types['"+v.getType()+"']";
datatype="find_type(group,'"+v.getType()+"')";
has_fill_value=false;
is_vlen_or_conpound=true;
// comment="#";//unsupported for now
}
......@@ -547,19 +580,31 @@ public class SCGroupAdapter {
dimensions="\""+String.join("\", \"", v.dims)+"\"";
}
sourceCode.add("\n"+localIdent +comment+ String.format("def create_%s(self, group: nc.Dataset):",v.name.toLowerCase()));
String miss_declare="";
//compound or vlen cannot have fill_value
if(!is_vlen_or_conpound)
{
miss_declare=String.format(" missing_value=%s,",has_fill_value ? fill_value : "None");
}
sourceCode.add("\n"+localIdent +comment+ String.format("def create_%s(self, group: nc.Dataset,%s **attributes):",v.name.toLowerCase(),miss_declare ));
sourceCode.add(localIdent +comment+ pythonIndent+ String.format("\"\"\"create variable %s\"\"\"",v.name.toLowerCase()));
if(has_fill_value)
sourceCode.add(localIdent +comment+ pythonIndent + String.format("v = group.createVariable(varname=\"%s\", datatype=%s, dimensions=(%s), fill_value=%s)",v.name,datatype,dimensions,fill_value));
else
sourceCode.add(localIdent +comment+ pythonIndent + String.format("v = group.createVariable(varname=\"%s\", datatype=%s, dimensions=(%s))",v.name,datatype,dimensions));
sourceCode.add(localIdent +comment+ pythonIndent + String.format("v = group.createVariable(varname=\"%s\", datatype=%s, dimensions=(%s)%s)",v.name,datatype,dimensions,is_vlen_or_conpound? "" : ", fill_value=missing_value"));
for(Attribute att: v.attributes)
{
if(att.getName().compareTo("missing_value")==0)
//skip
continue;
if (att.getName().compareTo("_FillValue")!=0)
writeAttribute(sourceCode,att,localIdent, "v");
}
//write parameter attributes
sourceCode.add("\n"+localIdent +comment+ pythonIndent + String.format("#write attributes passed as argument"));
sourceCode.add(localIdent +comment+ pythonIndent + String.format("for key, value in attributes.items():"));
sourceCode.add(localIdent +comment+ pythonIndent+pythonIndent + String.format("v.setncattr(key, value)"));
sourceCode.add(localIdent +comment+ pythonIndent + "return v\n");
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment