Commit 39ec629a authored by mickael's avatar mickael

more explicit column naming for overview

parent 136a618f
......@@ -711,7 +711,7 @@ def writeAllData(datemonth, picklerPath, satellite='a', showData=True):
############## PART FOR IPF ###################
##################################################
def writeAllDataIPF(ncfolder,pngfolder,version, picklerPath,start_date,end_date,satellite='a',hbaseIndex=None):
def writeAllDataIPF(ncfolder,pngfolder,version, picklerPath,start_date,end_date,satellite='a',calvalIndex=None):
#NCFiles
ncBaseList, ncFullPathList = getListFromNCOnHardDriveBlock(ncfolder, satellite)
......@@ -728,13 +728,13 @@ def writeAllDataIPF(ncfolder,pngfolder,version, picklerPath,start_date,end_date,
dictdbcolnames[satellite]['real_V2']+version,
dictdbcolnames[satellite]['full_V2']+version]
indexedFullPathList = getPathListInIndexHbaseBlock(start_date,end_date,satellite,ql_list,hbaseIndex)
indexedFullPathList = getPathListInIndexHbaseBlock(start_date,end_date,satellite,ql_list)
pickle.dump(indexedFullPathList, open(picklerPath + "/" + version + "indexedFullPathList.p", "wb"))
indexedInPhoenixNCBaseList = getPathListInPhoenixBlock(start_date,end_date,satellite)
pickle.dump(indexedInPhoenixNCBaseList, open(picklerPath + "/" + version + "indexedInPhoenixNCBaseList.p", "wb"))
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = getNCFileNameFromCalvalIndexBlock(start_date,end_date, satellite)
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = getNCFileNameFromCalvalIndexBlock(start_date,end_date, satellite,calvalIndex)
pickle.dump(geolocalizedElementsFromCalvalIndex, open(picklerPath + "/" + version + "geolocalizedElementsFromCalvalIndex.p", "wb"))
pickle.dump(notGeolocalizedElementsFromCalvalIndex, open(picklerPath + "/" + version + "notGeolocalizedElementsFromCalvalIndex.p", "wb"))
......@@ -802,13 +802,16 @@ def getPathListInIndexHbaseBlock(start_date,end_date,satellite='a',ql_list=None,
elementsIndexedInDatabase[qlcolname].append(r[1][qlcolname])
return elementsIndexedInDatabase
def getNCFileNameFromCalvalIndexBlock(start_date,end_date, satellite='a'):
def getNCFileNameFromCalvalIndexBlock(start_date,end_date, satellite='a',calvalIndex=None):
#ci = hbr.CersatIndexClient()
if satellite.upper() == 'A':
index_name = 'FRED/S1A_WV_SL2_OCN/calval1d/v1'
if calvalIndex is None:
if satellite.upper() == 'A':
index_name = 'FRED/S1A_WV_SL2_OCN/calval1d/v1'
else:
index_name = 'FRED/S1B_WV_SL2_OCN/calval1d/v2'
else:
index_name = 'FRED/S1B_WV_SL2_OCN/calval1d/v2'
index_name=calvalIndex
params = ['cf:osw']
indexElementInCalvalList = list(ci.getIndexFiles(index_name, start=start_date, stop=end_date, columns=params))
......@@ -923,22 +926,22 @@ def createHTMLOverviewIPF(picklerPath, satellite,versionList):
outputfile.write('<TR class="mytitle">')
outputfile.write('<TH rowspan="2" ></TH>')
outputfile.write('<TH colspan="10" >Sentinel 1 L2</TH>')
outputfile.write('<TH colspan="8" >Sentinel 1 L2</TH>')
# LV2 Data
outputfile.write('</TR><TR class="mytitle">')
outputfile.write('<TH>Duplicate L2 NC files</TH>')
outputfile.write('<TH>Duplicate L2 elements indexed Phoenix</TH>')
outputfile.write('<TH>Nc Files --> Unique Fname indexed in Phoenix</TH>')
outputfile.write('<TH>Real_V2 <br>PNG Generation</TH>')
outputfile.write('<TH>Real_V2 <br>PNG Indexation</TH>')
#outputfile.write('<TH>Duplicate L2 elements indexed Phoenix</TH>')
#outputfile.write('<TH>Nc Files --> Unique Fname indexed in Phoenix</TH>')
outputfile.write('<TH>Real_V2 -> Generation des png par rapport aux fichiers nc du filesystem</TH>')
outputfile.write('<TH>Real_V2 -> Indexation des vignettes dans hbase</TH>')
# outputfile.write('<TH>Imaginary</TH>')
outputfile.write('<TH>Imaginary_V2 PNG Generation</TH>')
outputfile.write('<TH>Imaginary_V2 PNG Indexation</TH>')
outputfile.write('<TH>Imaginary_V2 -> Generation des png par rapport aux fichiers nc du filesystem</TH>')
outputfile.write('<TH>Imaginary_V2 -> Indexation des vignettes dans hbase</TH>')
# outputfile.write('<TH>Full</TH>')
outputfile.write('<TH>Full_V2 PNG Generation</TH>')
outputfile.write('<TH>Full_V2 PNG Indexation</TH>')
outputfile.write('<TH>Full_V2 -> Generation des png par rapport aux fichiers nc du filesystem</TH>')
outputfile.write('<TH>Full_V2 -> Indexation des vignettes dans hbase</TH>')
# LV1 Data
......@@ -964,23 +967,17 @@ def createHTMLOverviewIPF(picklerPath, satellite,versionList):
# Duplicate NC Files
outputfile.write(createHTMLDotStatusDuplicate(
len(ncBaseList), doublonsLength,'duplicateL2NCFiles' + satellite.upper() + currentVersion + '.txt'))
#outputfile.write('<TD><img class="image-icon" ')
# if doublonsLength>0:
# outputfile.write('src="images/red_dot.png"')
# else:
# outputfile.write('src="images/green_dot.png"')
#outputfile.write(' title="'+str(doublonsLength)+'/'+str(len(ncBaseList))+","+str(len(ncFullPathList))+'"></TD>')
indexedFullPathList = pickle.load( open(picklerPath + "/" + currentVersion + "indexedFullPathList.p", "rb"))
indexedInPhoenixNCBaseList = pickle.load( open(picklerPath + "/" + currentVersion + "indexedInPhoenixNCBaseList.p", "rb"))
indexedInPhoenixNCBaseSet = set(indexedInPhoenixNCBaseList)
#Duplicate in phoenix
duplicateInPhoenix = len(indexedInPhoenixNCBaseList) - len(indexedInPhoenixNCBaseSet)
outputfile.write(createHTMLDotStatusDuplicate(
len(indexedInPhoenixNCBaseList), duplicateInPhoenix,'duplicateElementsIndexedInPhoenix' + satellite.upper() + currentVersion + '.txt'))
#duplicateInPhoenix = len(indexedInPhoenixNCBaseList) - len(indexedInPhoenixNCBaseSet)
#outputfile.write(createHTMLDotStatusDuplicate(len(indexedInPhoenixNCBaseList), duplicateInPhoenix,'duplicateElementsIndexedInPhoenix' + satellite.upper() + currentVersion + '.txt'))
#Indexation in phoenix
outputfile.write(createHTMLDotStatus(len(ncFullPathList), len(indexedInPhoenixNCBaseSet),0,'listings.html?' + currentVersion + '_NCFilesInPhoenix_' + satellite.upper() ))
#outputfile.write(createHTMLDotStatus(len(ncFullPathList), len(indexedInPhoenixNCBaseSet),0,'listings.html?' + currentVersion + '_NCFilesInPhoenix_' + satellite.upper() ))
# Sentinel 1 Data L2
......@@ -1701,7 +1698,7 @@ def createHTMLOverview(picklerPath, satellite):
outputfile.write('</TR><TR class="mytitle">')
outputfile.write('<TH>L2 NC files sur le filesystem en doublon</TH>')
outputfile.write('<TH>L2 elements indexes en doublons dans Phoenix (difference LON/LAT)</TH>')
outputfile.write('<TH>Nombre de fichier indexes dans phoenix par rapport au nombre de fichier nc sur le fielesysteme en excluant les doublons</TH>')
outputfile.write('<TH>Nombre de fichier indexes dans phoenix par rapport au nombre de fichier nc sur le filesystem en excluant les doublons</TH>')
outputfile.write('<TH>Real_V2 -> Generation des png par rapport aux fichiers nc du filesystem</TH>')
outputfile.write('<TH>Real_V2 -> Indexation des vignettes dans hbase</TH>')
# outputfile.write('<TH>Imaginary</TH>')
......@@ -1714,9 +1711,9 @@ def createHTMLOverview(picklerPath, satellite):
# LV1 Data
outputfile.write('<TH>XML Files --> HBASE INDEX </TH>')
outputfile.write('<TH>Nombre de lignes dans l index hbase par rapport aux fichier L1 du filesystem</TH>')
outputfile.write('<TH>Roughness -> Generation des png par rapport aux fichiers nc du filesystem</TH>')
outputfile.write('<TH>Roughness -> Indexation</TH>')
outputfile.write('<TH>Roughness -> Indexation des vignettes dans hbase</TH>')
# WW3 Data
outputfile.write('<TH>L2 -> Index calval</TH>')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment