Commit 58a8926f authored by mickael's avatar mickael

upgrade for listing in bubble

parent fd9ba4e0
......@@ -706,9 +706,27 @@ def writeAllData(datemonth, picklerPath, satellite='a', showData=True):
datemonth, picklerPath, satellite)
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = writeNCFileNameFromCalvalIndex(
datemonth, picklerPath, satellite)
# if showData:
# showAllData(pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,L1XmlFullPathDict,l1RoughnessBaseList,l1RoughnessFullPathList,satellite)
def writeAllDataCFO(datemonth, picklerPath):
pngBaseList, pngFullPathList = writeListFromPNGOnHardDrive(
datemonth, picklerPath, satellite)
ncBaseList, ncFullPathList = writeListFromNCOnHardDrive(
datemonth, picklerPath, satellite)
#ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater = writeListFromWW3OnHardDrive(datemonth,picklerPath,satellite)
ww3NcBaseList, ww3NcFullPathList = writeListFromWW3OnHardDrive(
datemonth, picklerPath, satellite)
ncBaseFromTrack, ncFilesFromTrackOnWater = writeFilesFromTrack(
datemonth, picklerPath, satellite)
indexedFullPathList = writePathListInIndex(
datemonth, picklerPath, satellite)
indexedInPhoenixNCBaseList = writePathListInPhoenix(
datemonth, picklerPath, satellite)
l1XmlBaseList, l1XmlFullPathList, L1XmlFullPathDict = writeListFromLevel1XmlOnHardDrive(
datemonth, picklerPath, satellite)
l1RoughnessBaseList, l1RoughnessFullPathList, l1RoughnessFullPathDict = writeListFromLevel1RoughnessOnHardDrive(
datemonth, picklerPath, satellite)
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = writeNCFileNameFromCalvalIndex(
datemonth, picklerPath, satellite)
def retrieveListFromPNGOnHardDrive(datemonth, picklerPath):
pngBaseList = pickle.load(
......@@ -1032,25 +1050,30 @@ def writeDuplicate(inputList,outputPath,outputFileName):
os.rename(outputfilepath+".tmp", outputfilepath+".txt")
return outputfilepath
def writeMissing(originList,resultList,outputPath,outputFileName):
outputfilepath = os.path.join(outputPath,outputFileName)
def writeMissing(originList,resultList,outputPath,outputFileName,fullPathListing=None):
outputfilepath = os.path.join(outputPath,"Missing"+outputFileName)
outputfile = open(outputfilepath+".tmp", 'w')
tmpElementsList = list(set(originList)-set(resultList))
outputfile.write("Not processed elements:\n")
for currentElement in tmpElementsList:
if fullPathListing:
if currentElement in fullPathListing:
currentElement = fullPathListing[currentElement]
outputfile.write(currentElement + "\n")
os.rename(outputfilepath+".tmp", outputfilepath+".txt")
outputfilepath = os.path.join(outputPath,"Vanished"+outputFileName)
outputfile = open(outputfilepath+".tmp", 'w')
tmpElementsList = list(set(resultList) - set(originList))
outputfile.write("\nElements that Vanished:\n")
for currentElement in tmpElementsList:
if fullPathListing:
if currentElement in fullPathListing:
currentElement = fullPathListing[currentElement]
outputfile.write(currentElement + "\n")
outputfile.close()
#if os.path.exists(outputfilepath+".txt"):
# os.remove(outputfilepath+".txt")
os.rename(outputfilepath+".tmp", outputfilepath+".txt")
print outputfilepath + " Finished"
return outputfilepath
def listElementsInError(dateToUse, picklerPath, satellite='a'):
......@@ -1066,52 +1089,51 @@ def listElementsInError(dateToUse, picklerPath, satellite='a'):
# duplicateL2NCFiles
writeDuplicate(ncBaseList,path,'duplicateL2NCFiles' + satellite.upper() + dateToUse)
# duplicateElementsIndexedInPhoenix
writeDuplicate(indexedInPhoenixNCBaseList,path,'duplicateElementsIndexedInPhoenix' + satellite.upper() + dateToUse)
# MissingNCFilesInPhoenix
writeMissing(ncBaseList,indexedInPhoenixNCBaseList,path,'MissingNCFilesInPhoenix' + satellite.upper() + dateToUse)
# MissingL2RealPngGeneration
writeMissing(ncBaseList,pngBaseList[dictdbcolnames[satellite]['real_V2']],path,'MissingL2RealPngGeneration' + satellite.upper() + dateToUse)
writeMissing(ncBaseList,indexedInPhoenixNCBaseList,path,'NCFilesInPhoenix' + satellite.upper() + dateToUse, ncFullPathList)
# MissingL2PngRealGeneration
writeMissing(ncBaseList,pngBaseList[dictdbcolnames[satellite]['real_V2']],path,'L2PngRealGeneration' + satellite.upper() + dateToUse,ncFullPathList)
indexedFullPathList = retrievePathListInIndex(dateToUse, picklerPath)
# MissingL2RealPngIndexation
writeMissing(pngFullPathList[dictdbcolnames[satellite]['real_V2']],indexedFullPathList[dictdbcolnames[satellite]['real_V2']],path,'MissingL2RealPngIndexation' + satellite.upper() + dateToUse)
writeMissing(pngFullPathList[dictdbcolnames[satellite]['real_V2']],indexedFullPathList[dictdbcolnames[satellite]['real_V2']],path,'L2PngRealIndexation' + satellite.upper() + dateToUse)
# MissingL2PngImaginaryGeneration
writeMissing(ncBaseList,pngBaseList[dictdbcolnames[satellite]['imaginary_V2']],path,'MissingL2PngImaginaryGeneration' + satellite.upper() + dateToUse)
writeMissing(ncBaseList,pngBaseList[dictdbcolnames[satellite]['imaginary_V2']],path,'L2PngImaginaryGeneration' + satellite.upper() + dateToUse,ncFullPathList)
# MissingL2PngImaginaryIndexation
writeMissing(pngFullPathList[dictdbcolnames[satellite]['imaginary_V2']],indexedFullPathList[dictdbcolnames[satellite]['imaginary_V2']],path,'MissingL2PngImaginaryIndexation' + satellite.upper() + dateToUse)
writeMissing(pngFullPathList[dictdbcolnames[satellite]['imaginary_V2']],indexedFullPathList[dictdbcolnames[satellite]['imaginary_V2']],path,'L2PngImaginaryIndexation' + satellite.upper() + dateToUse)
# MissingL2PngFullGeneration
writeMissing(ncBaseList,pngBaseList[dictdbcolnames[satellite]['full_V2']],path,'MissingL2PngFullGeneration' + satellite.upper() + dateToUse)
writeMissing(ncBaseList,pngBaseList[dictdbcolnames[satellite]['full_V2']],path,'L2PngFullGeneration' + satellite.upper() + dateToUse, ncFullPathList)
# MissingL2PngFullIndexation
writeMissing(pngFullPathList[dictdbcolnames[satellite]['full_V2']],indexedFullPathList[dictdbcolnames[satellite]['full_V2']],path,'MissingL2PngFullIndexation' + satellite.upper() + dateToUse)
writeMissing(pngFullPathList[dictdbcolnames[satellite]['full_V2']],indexedFullPathList[dictdbcolnames[satellite]['full_V2']],path,'L2PngFullIndexation' + satellite.upper() + dateToUse)
# MissingXmlIndexation
l1XmlBaseList, l1XmlFullPathList, l1XmlFullPathDict = retrieveListFromLevel1XmlOnHardDrive(dateToUse, picklerPath)
writeMissing(l1XmlFullPathList,indexedFullPathList[dictdbcolnames['L1']],path,'MissingXmlIndexation' + satellite.upper() + dateToUse)
writeMissing(l1XmlFullPathList,indexedFullPathList[dictdbcolnames['L1']],path,'XmlIndexation' + satellite.upper() + dateToUse)
# MissingRoughnessGeneration
l1RoughnessBaseList, l1RoughnessFullPathList, l1RoughnessFullPathDict = retrieveListFromLevel1RoughnessOnHardDrive(dateToUse, picklerPath)
writeMissing(l1XmlFullPathList,l1RoughnessFullPathList,path,'MissingRoughnessGeneration' + satellite.upper() + dateToUse)
writeMissing(l1XmlBaseList,l1RoughnessBaseList,path,'RoughnessGeneration' + satellite.upper() + dateToUse, l1XmlFullPathDict)
# MissingRoughnessIndexation
writeMissing(l1RoughnessFullPathList,indexedFullPathList[dictdbcolnames["roughness"]],path,'MissingRoughnessIndexation' + satellite.upper() + dateToUse)
writeMissing(l1RoughnessFullPathList,indexedFullPathList[dictdbcolnames["roughness"]],path,'RoughnessIndexation' + satellite.upper() + dateToUse)
#WW3 Part
# MissingL2InCalvalIndex
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = retrieveListGeolocalizedElementsFromCalvalIndex(dateToUse, picklerPath)
fullElementsFromCalvalIndex = geolocalizedElementsFromCalvalIndex + notGeolocalizedElementsFromCalvalIndex
writeMissing(ncBaseList,fullElementsFromCalvalIndex,path,'MissingL2InCalvalIndex' + satellite.upper() + dateToUse)
writeMissing(ncBaseList,fullElementsFromCalvalIndex,path,'L2InCalvalIndex' + satellite.upper() + dateToUse, ncFullPathList)
# MissingL2InTrackfile
ncBaseFromTrack, ncBaseFromTrackOnWater = retrieveFilesFromTrack(dateToUse, picklerPath)
writeMissing(geolocalizedElementsFromCalvalIndex,ncBaseFromTrack,path,'MissingL2InTrackfile' + satellite.upper() + dateToUse)
writeMissing(geolocalizedElementsFromCalvalIndex,ncBaseFromTrack,path,'L2InTrackfile' + satellite.upper() + dateToUse, ncFullPathList)
# MissingWW3NCFiles
ww3NcBaseList, ww3NcFullPathList = retrieveListFromWW3OnHardDrive(dateToUse, picklerPath)
writeMissing(ncBaseFromTrackOnWater,ww3NcBaseList,path,'MissingWW3NCFiles' + satellite.upper() + dateToUse)
writeMissing(ncBaseFromTrackOnWater,ww3NcBaseList,path,'WW3NCFiles' + satellite.upper() + dateToUse)
# MissingL2PngWW3Generation
writeMissing(ww3NcBaseList,pngBaseList[dictdbcolnames[satellite]['ww3_V2']],path,'MissingL2PngWW3Generation' + satellite.upper() + dateToUse)
writeMissing(ww3NcBaseList,pngBaseList[dictdbcolnames[satellite]['ww3_V2']],path,'L2PngWW3Generation' + satellite.upper() + dateToUse,ww3NcFullPathList)
# MissingL2PngWW3Indexation
writeMissing(pngFullPathList[dictdbcolnames[satellite]['ww3_V2']],indexedFullPathList[dictdbcolnames[satellite]['ww3_V2']],path,'MissingL2PngWW3Indexation' + satellite.upper() + dateToUse)
writeMissing(pngFullPathList[dictdbcolnames[satellite]['ww3_V2']],indexedFullPathList[dictdbcolnames[satellite]['ww3_V2']],path,'L2PngWW3Indexation' + satellite.upper() + dateToUse)
def createHTMLOutputGraph(dateToUse, picklerPath, satellite='a'):
......@@ -1233,8 +1255,12 @@ def createHTMLOutputGraph(dateToUse, picklerPath, satellite='a'):
os.rename(tmpoutputfilepath, outputfilepath)
def createHTMLDotStatus(inputLen, outputLen, duplicateLen=0):
resultString = ('<TD><img class="image-icon" ')
def createHTMLDotStatus(inputLen, outputLen, duplicateLen=0,fileLink=None):
resultString = '<TD>'
if fileLink:
resultString += ('<a href="'+fileLink+'" target="_blank">')
resultString += ('<img class="image-icon" ')
if outputLen == (inputLen - duplicateLen):
resultString += 'src="images/green_dot.png"'
# Case when there is missing elements
......@@ -1271,13 +1297,19 @@ def createHTMLDotStatus(inputLen, outputLen, duplicateLen=0):
resultString += 'src="images/brown_dot.png"'
missingElements = inputLen - outputLen - duplicateLen
resultString += ' title="' + str(outputLen) + '/' + str(
inputLen - duplicateLen) + '\n' + str(missingElements) + ' Missing' + '"></TD>'
inputLen - duplicateLen) + '\n' + str(missingElements) + ' Missing' + '">'
if fileLink:
resultString += ('</a>')
resultString += '</TD>'
return resultString
def createHTMLDotStatusDuplicate(inputLen, duplicateLen):
resultString = ('<TD><img class="image-icon" ')
def createHTMLDotStatusDuplicate(inputLen, duplicateLen,fileLink=None):
#resultString = ('<TD><img class="image-icon" ')
resultString = '<TD>'
if fileLink:
resultString += ('<a href="'+fileLink+'" target="_blank">')
resultString += '<img class="image-icon" '
if duplicateLen == 0:
resultString += 'src="images/green_dot.png"'
# Case when there is missing elements
......@@ -1295,7 +1327,10 @@ def createHTMLDotStatusDuplicate(inputLen, duplicateLen):
else:
resultString += 'src="images/brown_dot.png"'
resultString += ' title="' + str(duplicateLen) + '/' + str(
inputLen) + '\n' + str(duplicateLen) + ' Duplicates' + '"></TD>'
inputLen) + '\n' + str(duplicateLen) + ' Duplicates' + '">'
if fileLink:
resultString += ('</a>')
resultString += '</TD>'
return resultString
......@@ -1334,6 +1369,7 @@ def createHTMLOverview(picklerPath, satellite):
outputfile.write('<TABLE>')
# Generate head of the table
outputfile.write('<TR class="mytitle">')
outputfile.write('<TD><img class="image-icon" title="Everything allright" src="Bobarcade.png" height="50px"></TD>')
outputfile.write(
'<TD><img class="image-icon" title="Everything allright" src="images/green_dot.png"></TD><TD>Everything processed</TD>')
outputfile.write(
......@@ -1351,6 +1387,7 @@ def createHTMLOverview(picklerPath, satellite):
outputfile.write('<TD><img class="image-icon" title="More than 1% files And Less than 10% files Exceeding" src="images/bluesky_dot.png"></TD><TD>More than 1% files And Less than 10% files Exceeding</TD>')
outputfile.write(
'<TD><img class="image-icon" title="More than 10% files Exceeding" src="images/blue_dot.png"></TD><TD>More than 10% files Exceeding</TD>')
outputfile.write('<TD></TD><TD></TD><TD><img class="image-icon" title="Everything allright" src="Bubarcade.png" height="50px"></TD>')
outputfile.write('</TR>')
outputfile.write('</TABLE>')
......@@ -1382,7 +1419,7 @@ def createHTMLOverview(picklerPath, satellite):
# LV1 Data
outputfile.write('<TH>XML Files</TH>')
outputfile.write('<TH>XML Files --> HBASE INDEX </TH>')
outputfile.write('<TH>Roughness PNG Generation</TH>')
outputfile.write('<TH>Roughness PNG Indexation</TH>')
......@@ -1417,8 +1454,7 @@ def createHTMLOverview(picklerPath, satellite):
# Duplicate NC Files
outputfile.write(createHTMLDotStatusDuplicate(
len(ncBaseList), doublonsLength))
len(ncBaseList), doublonsLength,'duplicateL2NCFiles' + satellite.upper() + dateToUse + '.txt'))
#outputfile.write('<TD><img class="image-icon" ')
# if doublonsLength>0:
# outputfile.write('src="images/red_dot.png"')
......@@ -1433,11 +1469,11 @@ def createHTMLOverview(picklerPath, satellite):
duplicateInPhoenix = len(
indexedInPhoenixNCBaseList) - len(indexedInPhoenixNCBaseSet)
outputfile.write(createHTMLDotStatusDuplicate(
len(indexedInPhoenixNCBaseList), duplicateInPhoenix))
len(indexedInPhoenixNCBaseList), duplicateInPhoenix,'duplicateElementsIndexedInPhoenix' + satellite.upper() + dateToUse + '.txt'))
#Indexation in phoenix
outputfile.write(createHTMLDotStatus(
len(ncFullPathList), len(indexedInPhoenixNCBaseSet)))
len(ncFullPathList), len(indexedInPhoenixNCBaseSet),0,'listings.html?' + dateToUse + '_NCFilesInPhoenix_' + satellite.upper() ))
# Sentinel 1 Data L2
......@@ -1447,9 +1483,9 @@ def createHTMLOverview(picklerPath, satellite):
# for currentType in ['real','real_V2','imaginary','imaginary_V2','full','full_V2']:
for currentType in ['real_V2', 'imaginary_V2', 'full_V2']:
outputfile.write(createHTMLDotStatus(len(ncBaseList), len(
pngFullPathList[dictdbcolnames[satellite][currentType]])))
pngFullPathList[dictdbcolnames[satellite][currentType]]),0,'listings.html?' + dateToUse +"_L2Png"+currentType.capitalize()[:-3]+"Generation" +'_'+ satellite.upper()))
outputfile.write(createHTMLDotStatus(len(pngFullPathList[dictdbcolnames[satellite][currentType]]), len(
indexedFullPathList[dictdbcolnames[satellite][currentType]]), doublonsLength))
indexedFullPathList[dictdbcolnames[satellite][currentType]]), doublonsLength,'listings.html?' + dateToUse +"_L2Png"+currentType.capitalize()[:-3]+"Indexation" +'_'+ satellite.upper()))
# Sentinel 1 Data Lv1
......@@ -1457,23 +1493,23 @@ def createHTMLOverview(picklerPath, satellite):
dateToUse, picklerPath)
# L1 XML Indexation in hbase
outputfile.write(createHTMLDotStatus(len(l1XmlFullPathList), len(
indexedFullPathList[dictdbcolnames["L1"]])))
indexedFullPathList[dictdbcolnames["L1"]]),0,'listings.html?' + dateToUse +'_XmlIndexation_' + satellite.upper()))
l1RoughnessBaseList, l1RoughnessFullPathList, l1RoughnessFullPathDict = retrieveListFromLevel1RoughnessOnHardDrive(
dateToUse, picklerPath)
# L1 XML --> Roughness png
outputfile.write(createHTMLDotStatus(
len(l1RoughnessFullPathList), len(l1XmlFullPathList)))
len(l1XmlFullPathList),len(l1RoughnessFullPathList),0,'listings.html?' + dateToUse+'_RoughnessGeneration_' + satellite.upper()))
# Roughness indexation in hbase
outputfile.write(createHTMLDotStatus(len(
indexedFullPathList[dictdbcolnames["roughness"]]), len(l1RoughnessFullPathList)))
outputfile.write(createHTMLDotStatus(
len(l1RoughnessFullPathList), len(indexedFullPathList[dictdbcolnames["roughness"]]),0,'listings.html?' + dateToUse + '_RoughnessIndexation_' + satellite.upper()))
# WW3 Data
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = retrieveListGeolocalizedElementsFromCalvalIndex(
dateToUse, picklerPath)
# Difference between Tiff files and calval index
outputfile.write(createHTMLDotStatus(len(ncBaseList), len(
geolocalizedElementsFromCalvalIndex) + len(notGeolocalizedElementsFromCalvalIndex), doublonsLength))
geolocalizedElementsFromCalvalIndex) + len(notGeolocalizedElementsFromCalvalIndex), doublonsLength,'listings.html?' + dateToUse + '_L2InCalvalIndex_' + satellite.upper()))
ncBaseFromTrack, ncBaseFromTrackOnWater = retrieveFilesFromTrack(
dateToUse, picklerPath)
ww3NcBaseList, ww3NcFullPathList = retrieveListFromWW3OnHardDrive(
......@@ -1481,16 +1517,16 @@ def createHTMLOverview(picklerPath, satellite):
# Difference between gelocalized Elements in calval index and NC files in track on water
outputfile.write(createHTMLDotStatus(
len(geolocalizedElementsFromCalvalIndex), len(ncBaseFromTrack)))
len(geolocalizedElementsFromCalvalIndex), len(ncBaseFromTrack),0,'listings.html?' + dateToUse + '_L2InTrackfile_' + satellite.upper()))
# Difference NC files in tracks on water and Generated NC
outputfile.write(createHTMLDotStatus(
len(ncBaseFromTrackOnWater), len(ww3NcBaseList)))
len(ncBaseFromTrackOnWater), len(ww3NcBaseList),0,'listings.html?' + dateToUse + '_WW3NCFiles_' + satellite.upper()))
# Difference NC files and Generated PNG
outputfile.write(createHTMLDotStatus(len(ww3NcBaseList), len(
pngBaseList[dictdbcolnames[satellite]['ww3_V2']])))
pngBaseList[dictdbcolnames[satellite]['ww3_V2']]),0,'listings.html?' + dateToUse + '_L2PngWW3Generation_' + satellite.upper()))
# Difference NC files Generated PNG and indexed in Hbase
outputfile.write(createHTMLDotStatus(len(pngBaseList[dictdbcolnames[satellite]['ww3_V2']]), len(
indexedFullPathList[dictdbcolnames[satellite]['ww3_V2']])))
indexedFullPathList[dictdbcolnames[satellite]['ww3_V2']]),0,'listings.html?' + dateToUse + '_L2PngWW3Indexation_' + satellite.upper()))
outputfile.write('</TR>')
......@@ -1566,5 +1602,6 @@ if __name__ == "__main__":
createHTMLOutput(args.monthToProcess, picklerPath, args.satellite)
if args.function == 'writealldata':
writeAllData(args.monthToProcess, picklerPath, args.satellite)
listElementsInError(args.monthToProcess, picklerPath, args.satellite)
sys.exit(0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment