Commit 8a3671ce authored by mickael's avatar mickael

evolutions to incorporate phoenix

parent 0de26eb9
......@@ -10,7 +10,6 @@ from shapely.geometry import Polygon
import pandas as pd
from collections import defaultdict
import calendar
import datetime
from dateutil.relativedelta import relativedelta
import os
import re
......@@ -20,6 +19,8 @@ import numpy as np
import json
import collections
import time
#sys.path.append('/home/mlebars/workspace/git/cerberecontrib-cfosat')
sys.path.append('/home/satwave/sources_en_exploitation/cerbere')
......@@ -27,6 +28,12 @@ sys.path.append('/home/satwave/sources_en_exploitation/cerbere')
sys.path.append('/home/satwave/sources_en_exploitation/ceraux/ceraux')
import landmask
import phoenixdb
import phoenixdb.cursor
indexPhoenix = "INDEX_COLLECTIONFILES_PHX3"
urlDatabasePhoenix = "http://br156-175:8765"
ci = hbr.CersatIndexClient()
index_name='S1AWV.v3'
......@@ -41,6 +48,7 @@ platformB='S1B'
picklerPathA='/home/cercache/users/mlebars/output_test/checkgeneration_result/picklers/sat-a'
picklerPathB='/home/cercache/users/mlebars/output_test/checkgeneration_result/picklers/sat-b'
landmaskPath = '/home/satwave/sources_en_exploitation/fichier_references_cotes/PREVIMER_WW3-GLOBAL-30MIN_20160530T00Z.nc'
dictdbcolnames = {'L1':'cf:fpath',
'roughness':'cf:ql:l1_roughness',
......@@ -86,6 +94,43 @@ def getPathListInIndex(month_to_check,satellite='a'):
elementsIndexedInDatabase[qlcolname].append(r[1][qlcolname])
return elementsIndexedInDatabase
#Retrieve data indexed in hbase
def getPathListInPhoenix(month_to_check,satellite='a'):
start_date = datetime.datetime.strptime(month_to_check, '%Y%m')
#The end_date is not also included with phoenix
end_date=start_date+relativedelta(months=+1)
#avoid the future days if month is not finished yet
if start_date > datetime.datetime.today():
start_date = datetime.datetime.today()
conn = phoenixdb.connect(urlDatabasePhoenix, autocommit=True)
cursor = conn.cursor()
myRequest = "SELECT FNAME FROM " + indexPhoenix
whereClause = " WHERE FDATEDT BETWEEN TO_TIMESTAMP('"+str(start_date)+"','yyyy-MM-dd') AND TO_TIMESTAMP('"+str(end_date)+"','yyyy-MM-dd')"
if satellite=='a':
whereClause += " AND SENSOR IN('S1A') "
elif satellite=='b':
whereClause += " AND SENSOR IN('S1B') "
myRequest += whereClause
print myRequest
cursor.execute(myRequest)
elementsIndexed = cursor.fetchall()
tmpTab=[]
#Transforme les tuple de fname en donnees simple en retirant l'extension
for tmpValue in elementsIndexed:
tmpTab.append(os.path.splitext(tmpValue[0])[0])
elementsIndexed = list(tmpTab)
return elementsIndexed
# Will search throught all the folder for the specified month
#The day before and after will be checked to get missing elements
#the first and the last day will be checked to avoid other month elements
......@@ -224,7 +269,7 @@ def getListFromWW3OnHardDrive(month_to_check,satellite='a',fullNCLatLonOverwrite
print("getListFromWW3OnHardDrive",month_to_check)
#mylandmask = landmask.Landmask(filename='/home/oo12/oo/modeles_previmer/ww3/GLOBAL-30MIN/best_estimate/2016/PREVIMER_WW3-GLOBAL-30MIN_20160530T00Z.nc')
mylandmask = landmask.Landmask(filename='/home/satwave/sources_en_exploitation/fichier_references_cotes/PREVIMER_WW3-GLOBAL-30MIN_20160530T00Z.nc')
mylandmask = landmask.Landmask(filename=landmaskPath)
ncFiles = []
......@@ -237,20 +282,20 @@ def getListFromWW3OnHardDrive(month_to_check,satellite='a',fullNCLatLonOverwrite
basedir='/home/cercache/project/mpc-sentinel1/data/colocation/sentinel-1'+satellite+'/sar-model/ww3spectra/'+year+month+'/netCDF_L2'
cursor=0
IncorrectLatLon=0
#IncorrectLatLon=0
latLonPicklePath = picklerPath+"/"+month_to_check+"www3NCLatLonList.p"
#latLonPicklePath = picklerPath+"/"+month_to_check+"www3NCLatLonList.p"
#If file does not exist or overwrite mode create a new dictionary else load data from pickle
if fullNCLatLonOverwrite or not os.path.exists(latLonPicklePath):
ncLatLon={}
else:
ncLatLon = pickle.load(open(latLonPicklePath,"rb"))
#if fullNCLatLonOverwrite or not os.path.exists(latLonPicklePath):
# ncLatLon={}
#else:
# ncLatLon = pickle.load(open(latLonPicklePath,"rb"))
for root, dirs, files in os.walk(basedir):
for file in files:
if file.startswith("ww3") and file.endswith(".nc"):
#print("reading",file)
#add the fileNameBase --> change ww3 to s1a to have the exact same name
filename = file.replace('ww3-',replacestring)
......@@ -258,31 +303,32 @@ def getListFromWW3OnHardDrive(month_to_check,satellite='a',fullNCLatLonOverwrite
ncFiles.append(m.group(1))
ncFullPath[m.group(1)]=os.path.join(root,file)
if m.group(1) in ncLatLon:
lat,lon = ncLatLon[m.group(1)].split('_')
else:
fileToRead = Dataset(os.path.join(root,file), 'r')
try:
lon = np.squeeze(fileToRead.variables['lon'][:])
lat = np.squeeze(fileToRead.variables['lat'][:])
ncLatLon[m.group(1)] = str(float(lat)) + '_' + str(float(lon))
except TypeError:
IncorrectLatLon+=1
#if m.group(1) in ncLatLon:
# lat,lon = ncLatLon[m.group(1)].split('_')
#else:
# fileToRead = Dataset(os.path.join(root,file), 'r')
# try:
# lon = np.squeeze(fileToRead.variables['lon'][:])
# lat = np.squeeze(fileToRead.variables['lat'][:])
# ncLatLon[m.group(1)] = str(float(lat)) + '_' + str(float(lon))
#
# except TypeError:
# IncorrectLatLon+=1
#Check if it's on the sea
try:
lon = float(lon)
lat = float(lat)
if not mylandmask.is_land(lon=lon,lat=lat):
ncFilesOnWater.append(m.group(1))
except TypeError:
IncorrectLatLon+=1
if IncorrectLatLon>0:
print("Fichiers avec lat/lon incorrects:",IncorrectLatLon)
#try:
# lon = float(lon)
# lat = float(lat)
# if not mylandmask.is_land(lon=lon,lat=lat):
# ncFilesOnWater.append(m.group(1))
#except TypeError:
# IncorrectLatLon+=1
#if IncorrectLatLon>0:
# print("Fichiers avec lat/lon incorrects:",IncorrectLatLon)
pickle.dump(ncLatLon,open(latLonPicklePath,"wb"))
return ncFiles,ncFullPath,ncFilesOnWater
#pickle.dump(ncLatLon,open(latLonPicklePath,"wb"))
#return ncFiles,ncFullPath,ncFilesOnWater
return ncFiles,ncFullPath
import glob
def getAllListNCFromMPCListing(outputdir,satellites=['S1A','S1B']):
......@@ -348,7 +394,11 @@ def getFilesFromTrack(month_to_check,satellite='a'):
if len(trackfileList)>0:
trackfile = trackfileList[0]
ncFilesFromTrack = []
ncFilesFromTrackOnWater = []
isFirstLine=True
IncorrectLatLon = 0
mylandmask = landmask.Landmask(landmaskPath)
if os.path.exists(trackfile):
with open(trackfile) as inputfile:
for currentline in inputfile:
......@@ -356,12 +406,26 @@ def getFilesFromTrack(month_to_check,satellite='a'):
if isFirstLine:
isFirstLine=False
continue
currentValue = currentline.split(" ")[4]
tmpSplit = currentline.split(" ")
currentValue = tmpSplit[4]
lon = tmpSplit[2]
lat = tmpSplit[3]
#Remove extension if existing
if currentValue.endswith(".nc"):
ncFilesFromTrack.append(os.path.splitext(currentValue)[0])
else:
ncFilesFromTrack.append(currentValue)
return ncFilesFromTrack
currentValue = os.path.splitext(currentValue)[0]
ncFilesFromTrack.append(currentValue)
try:
lon = float(lon)
lat = float(lat)
if not mylandmask.is_land(lon=lon,lat=lat):
ncFilesFromTrackOnWater.append(currentValue)
except TypeError:
IncorrectLatLon+=1
if IncorrectLatLon>0:
print("Fichiers avec lat/lon incorrects:",IncorrectLatLon)
return ncFilesFromTrack,ncFilesFromTrackOnWater
def getNCFileNameFromCalvalIndex(month_to_check,satellite='a'):
......@@ -504,22 +568,33 @@ def writeListFromNCOnHardDrive(datemonth,picklerPath,satellite='a'):
return ncBaseList,ncFullPathList
def writeListFromWW3OnHardDrive(datemonth,picklerPath,satellite='a'):
ww3NcBaseList,ww3ncFullPathList,ww3NcFilesOnWater = getListFromWW3OnHardDrive(datemonth,satellite)
#ww3NcBaseList,ww3ncFullPathList,ww3NcFilesOnWater = getListFromWW3OnHardDrive(datemonth,satellite)
ww3NcBaseList,ww3ncFullPathList = getListFromWW3OnHardDrive(datemonth,satellite)
pickle.dump(ww3NcBaseList,open(picklerPath+"/"+datemonth+"ww3NcBaseList.p","wb"))
pickle.dump(ww3ncFullPathList,open(picklerPath+"/"+datemonth+"ww3ncFullPathList.p","wb"))
pickle.dump(ww3NcFilesOnWater,open(picklerPath+"/"+datemonth+"ww3NcFilesOnWater.p","wb"))
return ww3NcBaseList,ww3ncFullPathList,ww3NcFilesOnWater
#pickle.dump(ww3NcFilesOnWater,open(picklerPath+"/"+datemonth+"ww3NcFilesOnWater.p","wb"))
#return ww3NcBaseList,ww3ncFullPathList,ww3NcFilesOnWater
return ww3NcBaseList,ww3ncFullPathList
def writeFilesFromTrack(datemonth,picklerPath,satellite='a'):
ncBaseFromTrack = getFilesFromTrack(datemonth,satellite)
ncBaseFromTrack,ncFilesFromTrackOnWater = getFilesFromTrack(datemonth,satellite)
pickle.dump(ncBaseFromTrack,open(picklerPath+"/"+datemonth+"ncBaseFromTrack.p","wb"))
return ncBaseFromTrack
pickle.dump(ncFilesFromTrackOnWater,open(picklerPath+"/"+datemonth+"ncBaseFromTrackOnWater.p","wb"))
return ncBaseFromTrack,ncFilesFromTrackOnWater
def writePathListInIndex(datemonth,picklerPath,satellite='a'):
indexedFullPathList = getPathListInIndex(datemonth,satellite)
pickle.dump(indexedFullPathList,open(picklerPath+"/"+datemonth+"indexedFullPathList.p","wb"))
return indexedFullPathList
def writePathListInPhoenix(datemonth,picklerPath,satellite='a'):
indexedPhoenixNCBaseList = getPathListInPhoenix(datemonth,satellite)
print "len(indexedPhoenixNCBaseList):",len(indexedPhoenixNCBaseList)
pickle.dump(indexedPhoenixNCBaseList,open(picklerPath+"/"+datemonth+"indexedPhoenixNCBaseList.p","wb"))
print "after dump",picklerPath,datemonth
return indexedPhoenixNCBaseList
def writeListFromLevel1XmlOnHardDrive(datemonth,picklerPath,satellite='a'):
filePath,baseName,xmlFullPathDict = getListFromLevel1XmlOnHardDrive(datemonth,satellite)
pickle.dump(filePath,open(picklerPath+"/"+datemonth+"L1XmlFullPathList.p","wb"))
......@@ -543,9 +618,11 @@ def writeNCFileNameFromCalvalIndex(datemonth,picklerPath,satellite='a'):
def writeAllData(datemonth,picklerPath,satellite='a',showData=True):
pngBaseList,pngFullPathList = writeListFromPNGOnHardDrive(datemonth,picklerPath,satellite)
ncBaseList,ncFullPathList = writeListFromNCOnHardDrive(datemonth,picklerPath,satellite)
ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater = writeListFromWW3OnHardDrive(datemonth,picklerPath,satellite)
ncBaseFromTrack = writeFilesFromTrack(datemonth,picklerPath,satellite)
#ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater = writeListFromWW3OnHardDrive(datemonth,picklerPath,satellite)
ww3NcBaseList,ww3NcFullPathList = writeListFromWW3OnHardDrive(datemonth,picklerPath,satellite)
ncBaseFromTrack,ncFilesFromTrackOnWater = writeFilesFromTrack(datemonth,picklerPath,satellite)
indexedFullPathList = writePathListInIndex(datemonth,picklerPath,satellite)
indexedInPhoenixNCBaseList = writePathListInPhoenix(datemonth,picklerPath,satellite)
l1XmlBaseList,l1XmlFullPathList,L1XmlFullPathDict = writeListFromLevel1XmlOnHardDrive(datemonth,picklerPath,satellite)
l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict = writeListFromLevel1RoughnessOnHardDrive(datemonth,picklerPath,satellite)
geolocalizedElementsFromCalvalIndex,notGeolocalizedElementsFromCalvalIndex = writeNCFileNameFromCalvalIndex(datemonth,picklerPath,satellite)
......@@ -565,17 +642,23 @@ def retrieveListFromNCOnHardDrive(datemonth,picklerPath):
def retrieveListFromWW3OnHardDrive(datemonth,picklerPath):
ww3NcBaseList = pickle.load(open(picklerPath+"/"+datemonth+"ww3NcBaseList.p","rb"))
ww3NcFullPathList = pickle.load(open(picklerPath+"/"+datemonth+"ww3ncFullPathList.p","rb"))
ww3NcFilesOnWater = pickle.load(open(picklerPath+"/"+datemonth+"ww3NcFilesOnWater.p","rb"))
return ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater
#ww3NcFilesOnWater = pickle.load(open(picklerPath+"/"+datemonth+"ww3NcFilesOnWater.p","rb"))
#return ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater
return ww3NcBaseList,ww3NcFullPathList
def retrieveFilesFromTrack(datemonth,picklerPath):
ncBaseFromTrack = pickle.load(open(picklerPath+"/"+datemonth+"ncBaseFromTrack.p","rb"))
return ncBaseFromTrack
ncFilesFromTrackOnWater = pickle.load(open(picklerPath+"/"+datemonth+"ncBaseFromTrackOnWater.p","rb"))
return ncBaseFromTrack,ncFilesFromTrackOnWater
def retrievePathListInIndex(datemonth,picklerPath):
indexedFullPathList = pickle.load(open(picklerPath+"/"+datemonth+"indexedFullPathList.p","rb"))
return indexedFullPathList
def retrievePathListInPhoenix(datemonth,picklerPath):
indexedInPhoenixNCBaseList = pickle.load(open(picklerPath+"/"+datemonth+"indexedPhoenixNCBaseList.p","rb"))
return indexedInPhoenixNCBaseList
def retrieveListFromLevel1XmlOnHardDrive(datemonth,picklerPath,satellite='a'):
filePath = pickle.load(open(picklerPath+"/"+datemonth+"L1XmlFullPathList.p","rb"))
baseName = pickle.load(open(picklerPath+"/"+datemonth+"L1XmlBaseNameList.p","rb"))
......@@ -596,17 +679,19 @@ def retrieveListGeolocalizedElementsFromCalvalIndex(datemonth,picklerPath,satell
def retrieveAllData(datemonth,picklerPath,satellite='a',showData=True):
pngBaseList,pngFullPathList = retrieveListFromPNGOnHardDrive(datemonth,picklerPath)
ncBaseList,ncFullPathList = retrieveListFromNCOnHardDrive(datemonth,picklerPath)
ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater = retrieveListFromWW3OnHardDrive(datemonth,picklerPath)
#ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater = retrieveListFromWW3OnHardDrive(datemonth,picklerPath)
ww3NcBaseList,ww3NcFullPathList = retrieveListFromWW3OnHardDrive(datemonth,picklerPath)
ncBaseFromTrack = retrieveFilesFromTrack(datemonth,picklerPath)
indexedFullPathList = retrievePathListInIndex(datemonth,picklerPath)
indexedInPhoenixNCBaseList = retrievePathListInPhoenix(datemonth,picklerPath)
l1XmlBaseList,l1XmlFullPathList,l1XmlFullPathDict = retrieveListFromLevel1XmlOnHardDrive(datemonth,picklerPath)
l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict=retrieveListFromLevel1RoughnessOnHardDrive(datemonth,picklerPath)
geolocalizedElementsFromCalvalIndex,notGeolocalizedElementsFromCalvalIndex = retrieveListGeolocalizedElementsFromCalvalIndex(datemonth,picklerPath)
#if showData:
# showAllData(pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcFilesOnWater,ww3NcBaseList,ww3NcFullPathList,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,l1RoughnessBaseList,l1RoughnessFullPathList,satellite)
return pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,l1XmlFullPathDict,l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict,geolocalizedElementsFromCalvalIndex
return pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ncBaseFromTrack,indexedFullPathList,indexedInPhoenixNCBaseList,l1XmlBaseList,l1XmlFullPathList,l1XmlFullPathDict,l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict,geolocalizedElementsFromCalvalIndex
def showAllData(pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,l1RoughnessBaseList,l1RoughnessFullPathList,satellite='a'):
def showAllData(pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,l1RoughnessBaseList,l1RoughnessFullPathList,satellite='a'):
platform='S1A'
platformB='S1B'
......@@ -638,7 +723,7 @@ def showAllData(pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseL
#Angular or other stuff must be a better and cleaner solution but don't have time to learn them
def createHTMLOutput(datemonth,picklerPath,satellite='a'):
pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ww3NCFilesOnWater,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,l1XmlFullPathDict,l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict,geolocalizedElementsFromCalvalIndex = retrieveAllData(datemonth,picklerPath,satellite)
pngBaseList,pngFullPathList,ncBaseList,ncFullPathList,ww3NcBaseList,ww3NcFullPathList,ncBaseFromTrack,indexedFullPathList,l1XmlBaseList,l1XmlFullPathList,l1XmlFullPathDict,l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict,geolocalizedElementsFromCalvalIndex = retrieveAllData(datemonth,picklerPath,satellite)
checkDifferences = 0
......@@ -736,7 +821,7 @@ def createHTMLOutput(datemonth,picklerPath,satellite='a'):
outputfile.write('<TD>'+str(len(ncBaseList))+'</TD>')
outputfile.write('<TD>'+str(len(geolocalizedElementsFromCalvalIndex))+'</TD>')
outputfile.write('<TD>'+str(len(ncBaseFromTrack))+'</TD>')
outputfile.write('<TD>'+str(len(ww3NCFilesOnWater))+'</TD>')
#outputfile.write('<TD>'+str(len(ww3NCFilesOnWater))+'</TD>')
outputfile.write('<TD>'+str(len(ww3NcBaseList))+'</TD>')
outputfile.write('<TD>'+str(len(pngBaseList[dictdbcolnames[satellite]["ww3_V2"]]))+'</TD>')
outputfile.write('<TD>'+str(len(indexedFullPathList[dictdbcolnames[satellite]["ww3_V2"]]))+'</TD>')
......@@ -747,7 +832,7 @@ def createHTMLOutput(datemonth,picklerPath,satellite='a'):
#Errors between indexCalval and trackfile
outputfile.write(checkErrors(geolocalizedElementsFromCalvalIndex,"NCIndexedInCalval",ncBaseFromTrack,"NCInTrackFile",errorListPath,fileNamePrefix))
#Errors between Landmask NC and generated NCFiles
outputfile.write(checkErrors(ww3NCFilesOnWater,"OnWaterNCInTrackFile",ww3NcBaseList,"NC",errorListPath,fileNamePrefix))
#outputfile.write(checkErrors(ww3NCFilesOnWater,"OnWaterNCInTrackFile",ww3NcBaseList,"NC",errorListPath,fileNamePrefix))
#Errors Between Generated NCFiles and PNG
outputfile.write(checkErrors(ww3NcBaseList,"NC",pngBaseList[dictdbcolnames[satellite]["ww3_V2"]],"WW3PNG",errorListPath,fileNamePrefix))
#Errors between PNG and index
......@@ -826,7 +911,7 @@ def createHTMLOutputGraph(dateToUse,picklerPath,satellite='a'):
jsondata['children'][0]['children'][0]['children'][0]['status']=status
jsondata['children'][0]['children'][0]['children'][0]['description']=description
ww3NcBaseList,ww3NcFullPathList,ww3NcFilesOnWater = retrieveListFromWW3OnHardDrive(dateToUse,picklerPath)
ww3NcBaseList,ww3NcFullPathList = retrieveListFromWW3OnHardDrive(dateToUse,picklerPath)
#WW3 NC Files generation
status,description = comparelistLength(ncBaseFromTrack,ww3NcBaseList)
jsondata['children'][0]['children'][0]['children'][0]['children'][0]['status']=status
......@@ -834,9 +919,9 @@ def createHTMLOutputGraph(dateToUse,picklerPath,satellite='a'):
pngBaseList,pngFullPathList = retrieveListFromPNGOnHardDrive(dateToUse,picklerPath)
#WW3 PNG Generation
status,description = comparelistLength(ww3NcFilesOnWater,pngBaseList[dictdbcolnames[satellite]['ww3_V2']])
jsondata['children'][0]['children'][0]['children'][0]['children'][0]['children'][0]['status']=status
jsondata['children'][0]['children'][0]['children'][0]['children'][0]['children'][0]['description']=description
#status,description = comparelistLength(ww3NcFilesOnWater,pngBaseList[dictdbcolnames[satellite]['ww3_V2']])
#jsondata['children'][0]['children'][0]['children'][0]['children'][0]['children'][0]['status']=status
#jsondata['children'][0]['children'][0]['children'][0]['children'][0]['children'][0]['description']=description
indexedFullPathList = retrievePathListInIndex(dateToUse,picklerPath)
#WW3 PNG indexation
......@@ -890,6 +975,68 @@ def createHTMLOutputGraph(dateToUse,picklerPath,satellite='a'):
outputfile.write(json.dumps(jsondata))
os.rename(tmpoutputfilepath, outputfilepath)
def createHTMLDotStatus(inputLen,outputLen,duplicateLen=0):
resultString =('<TD><img class="image-icon" ')
if outputLen == (inputLen - duplicateLen):
resultString +='src="images/green_dot.png"'
#Case when there is missing elements
elif outputLen < (inputLen - duplicateLen):
if (inputLen == duplicateLen):
percentageInError = 100
else:
percentageInError = 100*(inputLen - duplicateLen - outputLen)/(inputLen - duplicateLen)
if percentageInError>30:
resultString+='src="images/red_dot.png"'
if percentageInError>10:
resultString+='src="images/lightred_dot.png"'
if percentageInError>1:
resultString+='src="images/orange_dot.png"'
else:
resultString+='src="images/yellow_dot.png"'
#Case When there is too much elements
elif outputLen > (inputLen - duplicateLen):
if (inputLen == duplicateLen):
percentageInError = 100
else:
percentageInError = 100*(outputLen - inputLen - duplicateLen)/(inputLen - duplicateLen)
if percentageInError>10:
resultString+='src="images/blue_dot.png"'
if percentageInError>1:
resultString+='src="images/bluesky_dot.png"'
else:
resultString+='src="images/cyan_dot.png"'
#Must not happen case,just in case
else:
resultString+='src="images/brown_dot.png"'
missingElements = inputLen - outputLen - duplicateLen
resultString+=' title="'+str(outputLen)+'/'+str(inputLen - duplicateLen)+'\n'+str(missingElements)+' Missing'+'"></TD>'
return resultString
def createHTMLDotStatusDuplicate(inputLen,duplicateLen):
resultString =('<TD><img class="image-icon" ')
if duplicateLen == 0:
resultString +='src="images/green_dot.png"'
#Case when there is missing elements
elif duplicateLen >0:
percentageInError = 100*(duplicateLen)/(inputLen)
if percentageInError>30:
resultString+='src="images/red_dot.png"'
if percentageInError>10:
resultString+='src="images/lightred_dot.png"'
if percentageInError>1:
resultString+='src="images/orange_dot.png"'
else:
resultString+='src="images/yellow_dot.png"'
#Must not happen case,just in case
else:
resultString+='src="images/brown_dot.png"'
resultString+=' title="'+str(duplicateLen)+'/'+str(inputLen)+'\n'+str(duplicateLen)+' Duplicates'+'"></TD>'
return resultString
def createHTMLOverview(picklerPath,satellite):
if satellite=='b':
......@@ -900,13 +1047,14 @@ def createHTMLOverview(picklerPath,satellite):
#satellite='a'
#picklerPath=picklerPathA
startDate = datetime.datetime.strptime('201401', '%Y%m')
startDate = datetime.datetime.strptime('201506', '%Y%m')
#startDate = datetime.datetime.strptime('201710', '%Y%m')
#endDate = datetime.datetime.strptime('201702', '%Y%m')
endDate = datetime.datetime.now()
outputfilepath = os.path.join(path,'generalOverview_'+satellite.upper()+".html")
outputfilepath = os.path.join(path,'generalOverview2_'+satellite.upper()+".html")
tmpoutputfilepath = os.path.join(path,'generalOverview_'+satellite.upper()+".tmp")
outputfile = open(tmpoutputfilepath,'w')
......@@ -921,10 +1069,18 @@ def createHTMLOverview(picklerPath,satellite):
#Generate head of the table
outputfile.write('<TR class="mytitle">')
outputfile.write('<TD><img class="image-icon" title="Everything allright" src="images/green_dot.png"></TD><TD>Everything processed</TD>')
outputfile.write('<TD><img class="image-icon" title="Some data may be not up to date" src="images/yellow_dot.png"></TD><TD>There is duplicate files all others files has been processed</TD>')
outputfile.write('<TD><img class="image-icon" title="Some input files no longer exist" src="images/orange_dot.png"></TD><TD>Some input Files has been processed but does not exist anymore</TD>')
outputfile.write('<TD><img class="image-icon" title="Some files have not been processed" src="images/red_dot.png"></TD><TD>Some elements have not been processed</TD>')
outputfile.write('<TD><img class="image-icon" title="Less than 1% files have not been processed" src="images/yellow_dot.png"></TD><TD>&lt;1% elements have not been processed</TD>')
outputfile.write('<TD><img class="image-icon" title="More than 1% files And Less than 10% files have not been processed" src="images/orange_dot.png"></TD><TD> &gt;1% &lt;10% elements have not been processed</TD>')
outputfile.write('<TD><img class="image-icon" title="More than 10% files And Less than 30% files have not been processed" src="images/lightred_dot.png"></TD><TD>&gt;10% &lt;30% elements have not been processed</TD>')
outputfile.write('<TD><img class="image-icon" title="More than 30% files have not been processed" src="images/red_dot.png"></TD><TD>&gt;30% elements have not been processed</TD>')
outputfile.write('</TR>')
outputfile.write('<TR class="mytitle">')
outputfile.write('<TD><img class="image-icon" title="UNKNOWN STATUS" src="images/brown_dot.png"></TD><TD>UNKNOWN STATUS</TD>')
outputfile.write('<TD><img class="image-icon" title="Less than 1% files Exceeding" src="images/cyan_dot.png"></TD><TD>Less than 1% files Exceeding</TD>')
outputfile.write('<TD><img class="image-icon" title="More than 1% files And Less than 10% files Exceeding" src="images/bluesky_dot.png"></TD><TD>More than 1% files And Less than 10% files Exceeding</TD>')
outputfile.write('<TD><img class="image-icon" title="More than 10% files Exceeding" src="images/blue_dot.png"></TD><TD>More than 10% files Exceeding</TD>')
outputfile.write('</TR>')
outputfile.write('</TABLE>')
......@@ -936,30 +1092,38 @@ def createHTMLOverview(picklerPath,satellite):
outputfile.write('<TH rowspan="2" ></TH>')
outputfile.write('<TH colspan="4" >Sentinel 1 L2</TH>')
outputfile.write('<TH colspan="2" >Sentinel 1 L1</TH>')
outputfile.write('<TH colspan="3" >WW3</TH>')
outputfile.write('<TH colspan="9" >Sentinel 1 L2</TH>')
outputfile.write('<TH colspan="3" >Sentinel 1 L1</TH>')
outputfile.write('<TH colspan="5" >WW3</TH>')
#LV2 Data
outputfile.write('</TR><TR class="mytitle">')
outputfile.write('<TH>Duplicate NC files</TH>')
#outputfile.write('<TH>Real</TH>')
outputfile.write('<TH>Real_V2</TH>')
outputfile.write('<TH>Duplicate L2 NC files</TH>')
outputfile.write('<TH>Duplicate L2 elements indexed Phoenix</TH>')
outputfile.write('<TH>Nc Files --> Unique Fname indexed in Phoenix</TH>')
outputfile.write('<TH>Real_V2 <br>PNG Generation</TH>')
outputfile.write('<TH>Real_V2 <br>PNG Indexation</TH>')
#outputfile.write('<TH>Imaginary</TH>')
outputfile.write('<TH>Imaginary_V2</TH>')
outputfile.write('<TH>Imaginary_V2 PNG Generation</TH>')
outputfile.write('<TH>Imaginary_V2 PNG Indexation</TH>')
#outputfile.write('<TH>Full</TH>')
outputfile.write('<TH>Full_V2</TH>')
outputfile.write('<TH>Full_V2 PNG Generation</TH>')
outputfile.write('<TH>Full_V2 PNG Indexation</TH>')
#LV1 Data
outputfile.write('<TH>XML Files</TH>')
outputfile.write('<TH>Roughness</TH>')
outputfile.write('<TH>Roughness PNG Generation</TH>')
outputfile.write('<TH>Roughness PNG Indexation</TH>')
#WW3 Data
outputfile.write('<TH>L2 -> calval</TH>')
outputfile.write('<TH>calval->NC</TH>')
outputfile.write('<TH>L2 -> Index calval</TH>')
outputfile.write('<TH>Geolocalized elements in Index calval-> Trackfile</TH>')
outputfile.write('<TH>TrackFiles element on water -> NC Files</TH>')
outputfile.write('<TH>NC Files -> PNG Generation</TH>')
#outputfile.write('<TH>NC->Index</TH>')
outputfile.write('<TH>NC->Index_V2</TH>')
outputfile.write('<TH>PNG Files -> Index_V2</TH>')
outputfile.write('</TR>')
......@@ -971,6 +1135,7 @@ def createHTMLOverview(picklerPath,satellite):
dateToUse=str(endDate.year)+str(endDate.month).zfill(2)
print dateToUse
pngBaseList,pngFullPathList = retrieveListFromPNGOnHardDrive(dateToUse,picklerPath)
ncBaseList,ncFullPathList = retrieveListFromNCOnHardDrive(dateToUse,picklerPath)
#doublons=[item for item, count in collections.Counter(ncBaseList).items() if count > 1]
doublonsLength=len(ncBaseList)-len(set(ncBaseList))
......@@ -981,108 +1146,62 @@ def createHTMLOverview(picklerPath,satellite):
outputfile.write('<TD><a href=monthly_graphic_overview.html?'+dateToUse+'_'+satellite.upper()+'>'+dateToUse+'</a></TD>')
#Duplicate NC Files
outputfile.write('<TD><img class="image-icon" ')
if doublonsLength>0:
outputfile.write('src="images/red_dot.png"')
else:
outputfile.write('src="images/green_dot.png"')
outputfile.write(' title="'+str(doublonsLength)+'/'+str(len(ncBaseList))+'"></TD>')
outputfile.write(createHTMLDotStatusDuplicate(len(ncBaseList),doublonsLength))
#outputfile.write('<TD><img class="image-icon" ')
#if doublonsLength>0:
# outputfile.write('src="images/red_dot.png"')
#else:
# outputfile.write('src="images/green_dot.png"')
#outputfile.write(' title="'+str(doublonsLength)+'/'+str(len(ncBaseList))+","+str(len(ncFullPathList))+'"></TD>')
indexedInPhoenixNCBaseList = retrievePathListInPhoenix(dateToUse,picklerPath)
indexedInPhoenixNCBaseSet = set(indexedInPhoenixNCBaseList)
#Duplicate in phoenix
duplicateInPhoenix = len(indexedInPhoenixNCBaseList)-len(indexedInPhoenixNCBaseSet)
outputfile.write(createHTMLDotStatusDuplicate(len(indexedInPhoenixNCBaseList),duplicateInPhoenix))
#Indexation in phoenix
outputfile.write(createHTMLDotStatus(len(ncFullPathList),len(indexedInPhoenixNCBaseSet)))
#Sentinel 1 Data L2
indexedFullPathList = retrievePathListInIndex(dateToUse,picklerPath)
#for currentType in ['real','real_V2','imaginary','imaginary_V2','full','full_V2']:
for currentType in ['real_V2','imaginary_V2','full_V2']:
outputfile.write('<TD><img class="image-icon" ')
if doublonsLength>0 and (len(indexedFullPathList[dictdbcolnames[satellite][currentType]])+doublonsLength==len(ncBaseList)):
outputfile.write('src="images/yellow_dot.png"')
elif len(indexedFullPathList[dictdbcolnames[satellite][currentType]])<len(ncBaseList):
outputfile.write('src="images/red_dot.png"')
elif len(indexedFullPathList[dictdbcolnames[satellite][currentType]])>len(ncBaseList):
outputfile.write('src="images/orange_dot.png"')
else:
outputfile.write('src="images/green_dot.png"')
missingElements = len(ncBaseList) - len(indexedFullPathList[dictdbcolnames[satellite][currentType]]) - doublonsLength
outputfile.write(' title="'+str(len(indexedFullPathList[dictdbcolnames[satellite][currentType]]))+'/'+str(len(ncBaseList))+'\n'+str(missingElements)+' Missing'+'"></TD>')
outputfile.write(createHTMLDotStatus(len(ncBaseList),len(pngFullPathList[dictdbcolnames[satellite][currentType]])))
outputfile.write(createHTMLDotStatus(len(pngFullPathList[dictdbcolnames[satellite][currentType]]),len(indexedFullPathList[dictdbcolnames[satellite][currentType]]),doublonsLength))
#Sentinel 1 Data Lv1
l1XmlBaseList,l1XmlFullPathList,l1XmlFullPathDict = retrieveListFromLevel1XmlOnHardDrive(dateToUse,picklerPath)
outputfile.write('<TD><img class="image-icon" ')
if len(l1XmlFullPathList)>len(indexedFullPathList[dictdbcolnames["L1"]]):
outputfile.write('src="images/red_dot.png"')
else:
outputfile.write('src="images/green_dot.png"')
missingElements = len(l1XmlFullPathList)-len(indexedFullPathList[dictdbcolnames["L1"]])
outputfile.write(' title="'+str(len(indexedFullPathList[dictdbcolnames["L1"]]))+'/'+str(len(l1XmlFullPathList))+'\n'+str(missingElements)+' Missing'+'"></TD>')
#L1 XML Indexation in hbase
outputfile.write(createHTMLDotStatus(len(l1XmlFullPathList),len(indexedFullPathList[dictdbcolnames["L1"]])))
l1RoughnessBaseList,l1RoughnessFullPathList,l1RoughnessFullPathDict = retrieveListFromLevel1RoughnessOnHardDrive(dateToUse,picklerPath)
outputfile.write('<TD><img class="image-icon" ')
if len(l1RoughnessFullPathList)>len(indexedFullPathList[dictdbcolnames["roughness"]]):
outputfile.write('src="images/red_dot.png"')
else:
outputfile.write('src="images/green_dot.png"')
missingElements = len(l1RoughnessFullPathList)-len(indexedFullPathList[dictdbcolnames["roughness"]])
outputfile.write(' title="'+str(len(indexedFullPathList[dictdbcolnames["roughness"]]))+'/'+str(len(l1RoughnessFullPathList))+'\n'+str(missingElements)+' Missing'+'"></TD>')
#L1 XML --> Roughness png
outputfile.write(createHTMLDotStatus(len(l1RoughnessFullPathList),len(l1XmlFullPathList)))
#Roughness indexation in hbase
outputfile.write(createHTMLDotStatus(len(indexedFullPathList[dictdbcolnames["roughness"]]),len(l1RoughnessFullPathList)))
#WW3 Data
geolocalizedElementsFromCalvalIndex,notGeolocalizedElementsFromCalvalIndex = retrieveListGeolocalizedElementsFromCalvalIndex(dateToUse,picklerPath)
#Difference between Tiff files and calval index
outputfile.write('<TD><img class="image-icon" ')
if doublonsLength>0 and (len(geolocalizedElementsFromCalvalIndex)+doublonsLength==len(ncBaseList)):
outputfile.write('src="images/yellow_dot.png"')
elif len(ncBaseList) > len(geolocalizedElementsFromCalvalIndex):
outputfile.write('src="images/red_dot.png"')
elif len(ncBaseList) < len(geolocalizedElementsFromCalvalIndex):