Commit ed06a1af authored by GROUAZEL's avatar GROUAZEL

Merge branch 'master' of gitvbise:wavetoolbox

parents 3e4e9a83 a6c3a8d4
This diff is collapsed.
import sys
mission_start_s1a = "20140101"
#filerepo = "/home3/homedir7/perso/satwave/sources_en_exploitation/wavetoolbox/"
filerepo = "/home3/homedir7/perso/mlebars/workspace/git/wavetoolbox/"
#sys.path.insert(0,'"/home3/homedir7/perso/satwave/sources_en_exploitation/mpc-sentinel/mpc-sentinel/mpcsentinellibs/indexation_in_hbase/')
sys.path.insert(0,'/home3/homedir7/perso/mlebars/workspace/git/mpc-sentinel/mpc-sentinel/mpcsentinellibs/indexation_in_hbase/')
import datetime
from dateutil.relativedelta import relativedelta
import hbaserequests as hbr
import datetime
from dateutil.relativedelta import relativedelta
import phoenixdb
import os
indexPhoenix = "INDEX_COLLECTIONFILES_PHX3"
urlDatabasePhoenix = "http://br156-175:8765"
picklerPathA='/home/cercache/users/mlebars/output_test/checkgeneration_result/picklers/sat-a'
picklerPathB='/home/cercache/users/mlebars/output_test/checkgeneration_result/picklers/sat-b'
platform='S1A'
platformB='S1B'
dictdbcolnames = {'L1':'cf:fpath',
'roughness':'cf:ql:l1_roughness',
'a':{'ww3_V2': 'cf:ql:'+platform+'_WV_OCN__2S_ww3'+'_V2',
'imaginary_V2':'cf:ql:'+platform+'_WV_OCN__2S_imaginarycrossspectra_'+platform.lower()+'_V2',
'real_V2':'cf:ql:'+platform+'_WV_OCN__2S_realcrossspectra_'+platform.lower()+'_V2',
'full_V2':'cf:ql:'+platform+'_WV_OCN__2S_ocean_swell_spectra_'+platform.lower()+'_V2'
},'b':{'ww3_V2': 'cf:ql:'+platformB+'_WV_OCN__2S_ww3'+'_V2',
'imaginary_V2':'cf:ql:'+platformB+'_WV_OCN__2S_imaginarycrossspectra_'+platformB.lower()+'_V2',
'real_V2':'cf:ql:'+platformB+'_WV_OCN__2S_realcrossspectra_'+platformB.lower()+'_V2',
'full_V2':'cf:ql:'+platformB+'_WV_OCN__2S_ocean_swell_spectra_'+platformB.lower()+'_V2'
}}
ci = hbr.CersatIndexClient()
def cmdlineparser(parser):
"""
define the options of the scripts and set func depending of the inputs
"""
#OBJCHOICES = ('WV', 'SAFE')
#parser.add_argument('-d', '--debug', action='store_true', default=False, help='Debug logs')
#parser.add_argument('--ww3colocprovenance', action='store', default=None,choices=['SATGLOB','REJEU2016','SENTINEL'], help='read the coloc WW3 from a specific directory = specific run [optional]')
#parser.add_argument('--indexname', action='store', default=None, help='give a different indexname for special purpose',required=False)
#parser.add_argument('-m','--mode', action='store',default=['both'],nargs=1, help='determine the storage mode hbase/phoenix/both by default it\'s both')
#parser.add_argument('data_type', type=str.upper, choices=OBJCHOICES)
parser.add_option("-b","--beginningdate",
action="store", type="string",
dest="begdate", metavar="string",
help="starting date YYYYMM [default=begining of mission]")
parser.add_option("-e","--endingdate",
action="store", type="string",
dest="enddate", metavar="string",
help="ending date YYYYMM [default=current day]")
parser.add_option("-s","--satellite",
action="store", type="string",
dest="satellite", metavar="string",
help="Mode [S1A,S1B,ALL]")
(options, args) = parser.parse_args()
return options
#Function create to list elements in phoenix and return key + elements of the key
#Retrieve data indexed in hbase
def getPathDictInPhoenix(month_to_check,satellite='a'):
start_date = datetime.datetime.strptime(month_to_check, '%Y%m')
#The end_date is not also included with phoenix
end_date=start_date+relativedelta(months=+1)
#avoid the future days if month is not finished yet
if start_date > datetime.datetime.today():
start_date = datetime.datetime.today()
conn = phoenixdb.connect(urlDatabasePhoenix, autocommit=True)
cursor = conn.cursor()
myRequest = "SELECT FNAME,SENSOR,FDATEDT,LON,LAT,FID FROM " + indexPhoenix
whereClause = " WHERE FDATEDT BETWEEN TO_TIMESTAMP('"+str(start_date)+"','yyyy-MM-dd') AND TO_TIMESTAMP('"+str(end_date)+"','yyyy-MM-dd')"
if satellite=='a':
whereClause += " AND SENSOR IN('S1A')"
elif satellite=='b':
whereClause += " AND SENSOR IN('S1B')"
myRequest += whereClause
print myRequest
cursor.execute(myRequest)
elementsIndexed = cursor.fetchall()
tmpTab={}
#Transforme les tuple de fname en donnees simple en retirant l'extension
for tmpValue in elementsIndexed:
#print tmpValue
tmpTab[os.path.splitext(tmpValue[0])[0]]= tmpValue
#elementsIndexed = list(tmpTab)
return tmpTab
#Check the presence of column in database
def getIndexedBoolleanDictInIndex(month_to_check,satellite='a'):
ql_list = [dictdbcolnames['L1'],dictdbcolnames['roughness'],
dictdbcolnames[satellite]['ww3_V2'],
dictdbcolnames[satellite]['imaginary_V2'],
dictdbcolnames[satellite]['real_V2'],
dictdbcolnames[satellite]['full_V2']]
print("Retrieve data from DB")
index_name='S1'+satellite.upper()+'WV.v3'
start_date = datetime.datetime.strptime(month_to_check, '%Y%m')
#The end_date is also included that's why one day is substracted
end_date=start_date+relativedelta(months=+1)-relativedelta(days=+1)
#avoid the future days if month is not finished yet
if start_date > datetime.datetime.today():
start_date = datetime.datetime.today()
elementsIndexedInDatabase = []
flist = list(ci.getIndexFiles(index_name, start=start_date.strftime('%Y%m%d'),stop=end_date.strftime('%Y%m%d'), columns=['cf:']))
MISSING_FDATE = 0
for r in flist:
tmpElement={}
for qlcolname in ql_list:
if qlcolname in r[1] and r[1][qlcolname] != 'unknown':
tmpElement[qlcolname] = r[1][qlcolname]
elementsIndexedInDatabase.append(tmpElement)
return elementsIndexedInDatabase
def upsertHbaseImagePresenceInPhoenix(monthToCheck,satellite):
elementDictInPhoenix = getPathDictInPhoenix(monthToCheck,satellite)
elementDictInHbase = getIndexedBoolleanDictInIndex(monthToCheck,satellite)
cpt=0
conn = phoenixdb.connect(urlDatabasePhoenix, autocommit=True)
cursor = conn.cursor()
for currentElement in elementDictInHbase:
#print currentElement
currentId = None
haveRoughness = False
haveWW3 = False
haveImaginary = False
haveReal = False
haveFull = False
#if dictdbcolnames['L1'] in currentElement:
# print "L1 is true"
if dictdbcolnames['roughness'] in currentElement:
haveRoughness = True
currentId = os.path.splitext(currentElement[dictdbcolnames['roughness']])[0].split('/')[-1].split('_')[-1].replace('-roughness','')
if dictdbcolnames[satellite]['ww3_V2'] in currentElement:
haveWW3 = True
currentId = os.path.splitext(currentElement[dictdbcolnames[satellite]['ww3_V2']])[0].split('/')[-1].split('_')[-1]
if dictdbcolnames[satellite]['imaginary_V2'] in currentElement:
haveImaginary = True
currentId = os.path.splitext(currentElement[dictdbcolnames[satellite]['imaginary_V2']])[0].split('/')[-1].split('_')[-1]
if dictdbcolnames[satellite]['real_V2'] in currentElement:
haveReal = True
currentId = os.path.splitext(currentElement[dictdbcolnames[satellite]['real_V2']])[0].split('/')[-1].split('_')[-1]
if dictdbcolnames[satellite]['full_V2'] in currentElement:
haveFull = True
currentId = os.path.splitext(currentElement[dictdbcolnames[satellite]['full_V2']])[0].split('/')[-1].split('_')[-1]
if currentId and currentId in elementDictInPhoenix:
myRequest= "UPSERT INTO "+indexPhoenix+"(SENSOR,FDATEDT,LON,LAT,FID, ROUGHNESSINDEXED,IMAGINARYINDEXED,REALINDEXED,FULLINDEXED,WW3INDEXED) VALUES("
myRequest+= "'"+elementDictInPhoenix[currentId][1]+"','"+str(elementDictInPhoenix[currentId][2].strftime("%Y-%m-%dT%H:%M:%S"))+"',"+str(elementDictInPhoenix[currentId][3])+","+str(elementDictInPhoenix[currentId][4])+",'"+str(elementDictInPhoenix[currentId][5])+"'"
myRequest+= ","+str(haveRoughness)+","+str(haveImaginary)+","+str(haveReal)+","+str(haveFull)+","+str(haveWW3)+")"
cursor.execute(myRequest)
cpt+=1
return cpt
def main():
from optparse import OptionParser
#Get the options--------
parser = OptionParser()
options = cmdlineparser(parser)
if options.enddate == None :
enddate = datetime.datetime.now()
print "noEndDate"
else:
enddate = datetime.datetime.strptime(options.enddate,'%Y%m')
if options.begdate == None:
startdate = mission_start_s1a
print "noStartDate"
else:
startdate = datetime.datetime.strptime(options.begdate,'%Y%m')
if options.satellite == "S1A":
satellite = ["a"]
elif options.satellite == "S1B":
satellite=["b"]
else:
satellite=["a","b"]
print satellite
print startdate,enddate
while startdate<=enddate:
currentMonth= str(startdate.year) + str(startdate.month).zfill(2)
for currentSatellite in satellite:
cpt = upsertHbaseImagePresenceInPhoenix(currentMonth,currentSatellite)
print cpt,'elements processed for',currentMonth,currentSatellite
startdate = startdate + relativedelta(months=+1)
sys.exit(0)
if __name__ == '__main__':
main()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment