Commit 4bd13263 authored by pm22d12's avatar pm22d12
Browse files

Fix #66: Real mtime of the file downloaded in https

Fix #69: Allow / ? @ : and # characters in passwords
parent 5475af12
......@@ -354,9 +354,12 @@ class Config:
class DCheck(object):
def getProtocolObject(self, protocolname, option=None):
def getProtocolObject(self, protocolname, path, option=None):
obj = None
if protocolname == "localpath":
if protocolname in ("localpath", "localmove", "localpointer") \
or protocolname == 'file' \
or (protocolname == '' and path.startswith('/')) \
or (protocolname == '' and path.startswith('./')):
obj = localpath.Protocol_localpath()
elif protocolname == "ftp":
kwargs = {}
......@@ -375,7 +378,7 @@ class DCheck(object):
obj = http_navy_rainglobal.Protocol_http_navy_rainglobal()
elif protocolname == "http_jma_jra_gribfinal":
obj = http_jma_jra_gribfinal.Protocol_http_jma_jra_gribfinal()
elif protocolname == "lslr_file":
elif protocolname == "lslr":
obj = lslr_file.Protocol_lslr_file()
elif protocolname == "https_opensearch":
obj = https_opensearch.Protocol_https_opensearch()
......@@ -385,7 +388,8 @@ class DCheck(object):
obj = webdav.Protocol_webdav()
elif protocolname == "ftps":
obj = ftps.Protocol_ftps(option)
elif protocolname == "https_directorylist":
#elif protocolname == "https_directorylist":
elif protocolname in ( "https", "http"):
classFileExtractor = https_directorylist.FileExtractor
redirector = None
if "Redirector" in option:
......@@ -558,7 +562,9 @@ class DCheck(object):
log.setLevel(config.logLevel)
log.debug("Config : " + config.constConfigString())
proto = self.getProtocolObject(config.protocol, config.protocol_option)
proto = self.getProtocolObject(config.protocol,
config.path,
config.protocol_option)
if proto is None:
raise DC_ConfigError("Unknown protocol : %s" % (config.protocol))
......
......@@ -336,7 +336,7 @@ class DCheckReportListingBuilder(IListingBuilder):
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeStringParameter(
self.__makeNumericParameter(
self.__download.configuration.source.port),
"SERVER_PORT",
"")
......
......@@ -141,8 +141,8 @@ class DownloadedFile(File):
self.__data_reader = data_reader
# 29/01/2018 PMT#28 : add attributs
#self.size = None
#self.mtime = None
self.filesize = None
self.mtime = None
self.downloadEndTime = None
self.__storageName = None
......@@ -1619,6 +1619,12 @@ class Download(threading.Thread):
downloaded_file.localFile = local_file
downloaded_file.isDownloaded = True
downloaded_mtime_file = self.__provider.getmtime()
if downloadChecksum is False:
downloaded_file.mtime = self.__provider.getmtime()
downloaded_file.filesize = self.__provider.getsize()
self._log.debug(f"Real downloaded mtime_file : {str(downloaded_file.mtime)},"
f"Real downloaded size_file : {str(downloaded_file.filesize)},")
stop_time = time.time()
# Gestion du checksum, dans le cas d'un fichier normal (pas un
......@@ -2628,9 +2634,11 @@ class Download(threading.Thread):
if downloaded_file.localFile.exist():
mtime = time.strftime(DATE_FORMAT, time.gmtime(downloaded_file.localFile.getTime()))
size = str(downloaded_file.localFile.getSize())
print(f"mtime fichier : {mtime}")
else:
mtime = source_file.mtime.strftime(DATE_FORMAT)
size = str(source_file.size)
print(f"mtime source : {mtime}")
msg_downloaded_file = messages.File(
name=downloaded_file.localFile.getName(short=True),
......@@ -2724,22 +2732,24 @@ class Download(threading.Thread):
if self.__monitoring \
and msg_level in ['INFO', 'WARNING']\
and not self.test:
self.__writeJsonFile(source_file)
self.__writeJsonFile(source_file, downloaded_file)
def __writeJsonFile(self, source_file):
def __writeJsonFile(self, source_file, downloaded_file):
mtime = source_file.mtime if downloaded_file.mtime is None else downloaded_file.mtime
size = source_file.size if downloaded_file.filesize is None else downloaded_file.filesize
nowDate = datetime.datetime.utcnow()
deltaTimePLD = source_file.mtime - source_file.sensingtime
deltaTimeLUD = nowDate - source_file.mtime
deltaTimePLD = mtime - source_file.sensingtime
deltaTimeLUD = nowDate - mtime
metrics = {
"filename": os.path.split(
source_file.filepath.split(FILE_URL_SEPARATOR)[0])[1],
"download": self.id,
"download_time": nowDate.strftime(JSON_DATE_FORMAT) ,
"sensing_time": source_file.sensingtime.strftime(JSON_DATE_FORMAT),
"modification_time": source_file.mtime.strftime(JSON_DATE_FORMAT),
"modification_time": mtime.strftime(JSON_DATE_FORMAT),
"production_latency_delay": deltaTimePLD.days * 86400 + deltaTimePLD.seconds,
"local_update_delay": deltaTimeLUD.days * 86400 + deltaTimeLUD.seconds,
"file_volume": source_file.size,
"file_volume": size,
}
tookedSemaphore = False
jsonFile = None
......
......@@ -209,3 +209,11 @@ class AbstractProvider(object):
def setOption(self, optionDict):
self.__option = optionDict
def getmtime(self):
""" get method for mtime, see method above for more info """
return None
def getsize(self):
""" get method for size, see method above for more info """
return None
......@@ -13,6 +13,7 @@ import urllib.request
import urllib.error
import urllib.parse
import http.cookiejar
import datetime
from downloader.scheduler.com.sys.File import File, READ_BUFFER
......@@ -28,6 +29,7 @@ class HttpsSession(object):
self.filename = None
self.filesize = 0
self.option = option
self.mtime = None
socket.setdefaulttimeout(self.timeout)
......@@ -61,6 +63,24 @@ class HttpsSession(object):
if (self.opener is not None):
flow = self.opener.open(remotepath)
downloadfile = open(localpath, 'wb')
self.filesize = int(flow.info()['Content-Length'])
if 'Last-Modified' in flow.info():
try:
# RFC 822, updated by RFC 1123
self.mtime = datetime.datetime.strptime(flow.info()['Last-Modified'], '%a, %d %b %Y %H:%M:%S GMT')
except:
try:
# ANSI C's asctime() format
self.mtime = datetime.datetime.strptime(flow.info()['Last-Modified'], '%c')
except:
try:
# RFC 850, obsoleted by RFC 1036
self.mtime = datetime.datetime.strptime(flow.info()['Last-Modified'], '%A, %d-%b-%y %H:%M:%S GMT')
except:
self.mtime = None
else:
self.mtime = None
self.filesize = int(flow.info()['Content-Length'])
# check real filename
if 'Content-Disposition' in flow.info() \
......@@ -96,6 +116,12 @@ class HttpsSession(object):
def getFilename(self):
return(self.filename)
def getFilemtime(self):
return(self.mtime)
def getFilesize(self):
return(self.filesize)
class HttpsProvider(AbstractProvider):
"""
......@@ -112,6 +138,8 @@ class HttpsProvider(AbstractProvider):
self.__password = None
self.__session = None
self.__option = None
self.__file_mtime = None
self.__file_size = None
server = property(lambda self: self.__server, None, None, None)
username = property(lambda self: self.__username, None, None, None)
......@@ -182,6 +210,11 @@ class HttpsProvider(AbstractProvider):
# externe eventuellement.
target_file = session.getFilename()
self.__file_mtime = session.getFilemtime()
self.__file_size = session.getFilesize()
try:
os.rename(tmp_localpath, target_file)
except OSError as msg:
......@@ -220,3 +253,9 @@ class HttpsProvider(AbstractProvider):
def setOption(self, optionDict):
self.__option = optionDict
def getmtime(self):
return self.__file_mtime
def getsize(self):
return self.__file_size
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment