Commit 795e30ef authored by mickael's avatar mickael

resolve pb with L1 xml/tiff

parent ab279a91
......@@ -531,11 +531,11 @@ def getListFromLevel1XmlOnHardDrive(month_to_check, satellite='a'):
# remove prefix and extension to have the same baseName
if file.startswith("s1") and file.endswith(".xml") and os.path.basename(root) == "annotation":
# add the fileNameBase
tiffFilePath.append(os.path.join(root.replace(
"/annotation", "/measurement"), file.replace('.xml', '.tiff')))
baseName.append(os.path.splitext(file)[0])
xmlFullPathDict[os.path.splitext(
file)[0]] = os.path.join(root, file)
currentValue = os.path.join(root.replace("/annotation", "/measurement"), file.replace('.xml', '.tiff'))
tiffFilePath.append(currentValue)
#xmlFullPathDict[os.path.splitext(file)[0]] = currentValue
xmlFullPathDict[os.path.splitext(file)[0]] = os.path.join(root, file)
currentdate = currentdate - relativedelta(days=+1)
return tiffFilePath, baseName, xmlFullPathDict
......@@ -707,7 +707,7 @@ def writeAllData(datemonth, picklerPath, satellite='a', showData=True):
geolocalizedElementsFromCalvalIndex, notGeolocalizedElementsFromCalvalIndex = writeNCFileNameFromCalvalIndex(
datemonth, picklerPath, satellite)
listElementsInError(args.monthToProcess, picklerPath, args.satellite)
listElementsInError(datemonth, picklerPath, satellite)
def writeAllDataCFO(datemonth, picklerPath):
pngBaseList, pngFullPathList = writeListFromPNGOnHardDrive(
......@@ -1051,16 +1051,20 @@ def writeDuplicate(inputList,outputPath,outputFileName):
outputfile.close()
os.rename(outputfilepath+".tmp", outputfilepath+".txt")
return outputfilepath
def writeMissing(originList,resultList,outputPath,outputFileName,fullPathListing=None):
# Write missing elements in 2 files
# replacingTab waiting a tab of tab to made replacement in written file used for L1 Tiff/xml
def writeMissing(originList,resultList,outputPath,outputFileName,fullPathListing=None,replacingTab=[]):
outputfilepath = os.path.join(outputPath,"Missing"+outputFileName)
outputfile = open(outputfilepath+".tmp", 'w')
tmpElementsList = list(set(originList)-set(resultList))
for currentElement in tmpElementsList:
if fullPathListing:
if currentElement in fullPathListing:
currentElement = fullPathListing[currentElement]
for currentReplacement in replacingTab:
currentElement = currentElement.replace(currentReplacement[0],currentReplacement[1])
outputfile.write(currentElement + "\n")
os.rename(outputfilepath+".tmp", outputfilepath+".txt")
......@@ -1072,6 +1076,8 @@ def writeMissing(originList,resultList,outputPath,outputFileName,fullPathListing
if fullPathListing:
if currentElement in fullPathListing:
currentElement = fullPathListing[currentElement]
for currentReplacement in replacingTab:
currentElement = currentElement.replace(currentReplacement[0],currentReplacement[1])
outputfile.write(currentElement + "\n")
outputfile.close()
os.rename(outputfilepath+".tmp", outputfilepath+".txt")
......@@ -1114,10 +1120,13 @@ def listElementsInError(dateToUse, picklerPath, satellite='a'):
# MissingXmlIndexation
l1XmlBaseList, l1XmlFullPathList, l1XmlFullPathDict = retrieveListFromLevel1XmlOnHardDrive(dateToUse, picklerPath)
writeMissing(l1XmlFullPathList,indexedFullPathList[dictdbcolnames['L1']],path,'XmlIndexation' + satellite.upper() + dateToUse)
#XML files are used to index but Tiff path are indexed in hbase,
# use Tiff filepath to compare but change them in xml filepath for the listing (replacingTab)
writeMissing(l1XmlFullPathList,indexedFullPathList[dictdbcolnames['L1']],path,'XmlIndexation' + satellite.upper() + dateToUse,replacingTab=[['/measurement/','/annotation/'],['.tiff','.xml']])
# MissingRoughnessGeneration
l1RoughnessBaseList, l1RoughnessFullPathList, l1RoughnessFullPathDict = retrieveListFromLevel1RoughnessOnHardDrive(dateToUse, picklerPath)
writeMissing(l1XmlBaseList,l1RoughnessBaseList,path,'RoughnessGeneration' + satellite.upper() + dateToUse, l1XmlFullPathDict)
writeMissing(l1XmlBaseList,l1RoughnessBaseList,path,'RoughnessGeneration' + satellite.upper() + dateToUse, l1XmlFullPathDict,replacingTab=[['/annotation/','/measurement/'],['.xml','.tiff']])
# MissingRoughnessIndexation
writeMissing(l1RoughnessFullPathList,indexedFullPathList[dictdbcolnames["roughness"]],path,'RoughnessIndexation' + satellite.upper() + dateToUse)
......@@ -1428,7 +1437,7 @@ def createHTMLOverview(picklerPath, satellite):
# WW3 Data
outputfile.write('<TH>L2 -> Index calval</TH>')
outputfile.write(
'<TH>Geolocalized elements in Index calval-> Trackfile</TH>')
'<TH>Geolocalized elements in Index calval (HBASE)-> Trackfile</TH>')
outputfile.write('<TH>TrackFiles element on water -> NC Files</TH>')
outputfile.write('<TH>NC Files -> PNG Generation</TH>')
# outputfile.write('<TH>NC->Index</TH>')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment