Commit 9cffe997 authored by MAISSIAT's avatar MAISSIAT Committed by BODERE
Browse files

Fix #66: Real mtime of the file downloaded in https

Fix #69: Allow / ? @ : and # characters in passwords
parent 53e3dd9b
......@@ -5,11 +5,9 @@ Fabien CADORET <fcadoret@br152-128.ifremer.fr>
Frederic PAUL <fpaul@hancock.ifremer.fr>
MAISSIAT <pierre.maissiat@partenaire-exterieur.ifremer.fr>
PMAISSIA <Pierre.Maissiat@partenaire-exterieur.ifremer.fr>
Pierre MAISSIAT <pmaissia@ananda.ifremer.fr>
Sylvain Herlédan <sylvain.herledan@shadowz.sytes.net>
Thibaut CHARLES <thibaut.charles@ifremer.fr>
bcauseur <bcauseur@sii.fr>
jfpiolle <jfpiolle@ifremer.fr>
medspi <medspi@br156-133.ifremer.fr>
pifgold <pifgold@free.fr>
pm22d12 <occulterai purees fonctionnais>
#
#
#
#
# ------------------------------------------------------------------
FROM continuumio/miniconda3
SHELL ["/bin/bash", "--login", "-c"]
ARG ROOTPATH=/tmp
ARG WORKSPACE=$ROOTPATH/workspace
ARG APPDATA=$ROOTPATH/appdata
RUN cd $ROOTPATH && \
git clone https://gitlab.ifremer.fr/downloader/downloader_daemon.git && \
conda env create -f downloader_daemon/environment.yaml && \
apt-get -y update && \
apt-get -y install gcc && \
conda init bash && \
rm -rf /var/cache/apk/*
WORKDIR $ROOTPATH/downloader_daemon
RUN conda clean -a && \
conda activate downloader && \
pip install -r requirements.txt --index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/pypi \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-public-snapshot/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-private-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-private-snapshot/simple && \
pip install .
# ------------------------------------------------------------------
EXPOSE 80
ENV DATAPATH=/home
ENV CONFIG_FILE=${APPDATA}/application.yaml OPTIONS="-f"
RUN mkdir -p ${WORKSPACE} && \
mkdir -p ${APPDATA} && \
mkdir -p /home/data && \
chmod 777 ${WORKSPACE} && \
chmod 777 ${APPDATA} && \
chmod 777 ${DATAPATH}
VOLUME [ "${WORKSPACE}" , "${APPDATA}" , "${DATAPATH}"]
ENTRYPOINT conda activate downloader && \
downloader -c ${CONFIG_FILE} ${OPTIONS}
CMD [ "/bin/bash" ]
......@@ -354,9 +354,12 @@ class Config:
class DCheck(object):
def getProtocolObject(self, protocolname, option=None):
def getProtocolObject(self, protocolname, path, option=None):
obj = None
if protocolname == "localpath":
if protocolname in ("localpath", "localmove", "localpointer") \
or protocolname == 'file' \
or (protocolname == '' and path.startswith('/')) \
or (protocolname == '' and path.startswith('./')):
obj = localpath.Protocol_localpath()
elif protocolname == "ftp":
kwargs = {}
......@@ -375,7 +378,7 @@ class DCheck(object):
obj = http_navy_rainglobal.Protocol_http_navy_rainglobal()
elif protocolname == "http_jma_jra_gribfinal":
obj = http_jma_jra_gribfinal.Protocol_http_jma_jra_gribfinal()
elif protocolname == "lslr_file":
elif protocolname == "lslr":
obj = lslr_file.Protocol_lslr_file()
elif protocolname == "https_opensearch":
obj = https_opensearch.Protocol_https_opensearch()
......@@ -383,9 +386,10 @@ class DCheck(object):
obj = sftp.Protocol_sftp()
elif protocolname == "webdav":
obj = webdav.Protocol_webdav()
elif protocolname[:4] == "ftps":
elif protocolname == "ftps":
obj = ftps.Protocol_ftps(option)
elif protocolname == "https_directorylist":
#elif protocolname == "https_directorylist":
elif protocolname in ( "https", "http"):
classFileExtractor = https_directorylist.FileExtractor
redirector = None
if "Redirector" in option:
......@@ -525,7 +529,7 @@ class DCheck(object):
if len(args) > 0:
url = args[0]
setConfigFromUrl(config, url)
#setConfigFromUrl(config, url)
if options.debug_mode:
config.debug = True
......@@ -558,7 +562,9 @@ class DCheck(object):
log.setLevel(config.logLevel)
log.debug("Config : " + config.constConfigString())
proto = self.getProtocolObject(config.protocol, config.protocol_option)
proto = self.getProtocolObject(config.protocol,
config.path,
config.protocol_option)
if proto is None:
raise DC_ConfigError("Unknown protocol : %s" % (config.protocol))
......
......@@ -316,12 +316,48 @@ class DCheckReportListingBuilder(IListingBuilder):
self.__download.configuration.source.opensearch_requestFormat),
"RESULT_FORMAT",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeStringParameter(
self.__download.configuration.source.protocol).lower(),
"PROTOCOL",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeDictParameter(
self.__download.configuration.source.protocol_option),
"PROTOCOL_OPTION",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeStringParameter(
self.__download.configuration.source.server),
"SERVER_ADDRESS",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeNumericParameter(
self.__download.configuration.source.port),
"SERVER_PORT",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeStringParameter(
self.__download.configuration.source.rootPath),
"PATH",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeStringParameter(
self.__download.configuration.source.login),
"AUTH_USERNAME",
"")
self.__writeLineDCheckFileConfig(
configFile,
self.__makeStringParameter(
self.__download.configuration.source.password),
"AUTH_PASSWORD",
"")
configFile.close()
......
......@@ -141,8 +141,8 @@ class DownloadedFile(File):
self.__data_reader = data_reader
# 29/01/2018 PMT#28 : add attributs
#self.size = None
#self.mtime = None
self.filesize = None
self.mtime = None
self.downloadEndTime = None
self.__storageName = None
......@@ -1619,6 +1619,12 @@ class Download(threading.Thread):
downloaded_file.localFile = local_file
downloaded_file.isDownloaded = True
downloaded_mtime_file = self.__provider.getmtime()
if downloadChecksum is False:
downloaded_file.mtime = self.__provider.getmtime()
downloaded_file.filesize = self.__provider.getsize()
self._log.debug(f"Real downloaded mtime_file : {str(downloaded_file.mtime)},"
f"Real downloaded size_file : {str(downloaded_file.filesize)},")
stop_time = time.time()
# Gestion du checksum, dans le cas d'un fichier normal (pas un
......@@ -2628,9 +2634,11 @@ class Download(threading.Thread):
if downloaded_file.localFile.exist():
mtime = time.strftime(DATE_FORMAT, time.gmtime(downloaded_file.localFile.getTime()))
size = str(downloaded_file.localFile.getSize())
print(f"mtime fichier : {mtime}")
else:
mtime = source_file.mtime.strftime(DATE_FORMAT)
size = str(source_file.size)
print(f"mtime source : {mtime}")
msg_downloaded_file = messages.File(
name=downloaded_file.localFile.getName(short=True),
......@@ -2724,22 +2732,24 @@ class Download(threading.Thread):
if self.__monitoring \
and msg_level in ['INFO', 'WARNING']\
and not self.test:
self.__writeJsonFile(source_file)
self.__writeJsonFile(source_file, downloaded_file)
def __writeJsonFile(self, source_file):
def __writeJsonFile(self, source_file, downloaded_file):
mtime = source_file.mtime if downloaded_file.mtime is None else downloaded_file.mtime
size = source_file.size if downloaded_file.filesize is None else downloaded_file.filesize
nowDate = datetime.datetime.utcnow()
deltaTimePLD = source_file.mtime - source_file.sensingtime
deltaTimeLUD = nowDate - source_file.mtime
deltaTimePLD = mtime - source_file.sensingtime
deltaTimeLUD = nowDate - mtime
metrics = {
"filename": os.path.split(
source_file.filepath.split(FILE_URL_SEPARATOR)[0])[1],
"download": self.id,
"download_time": nowDate.strftime(JSON_DATE_FORMAT) ,
"sensing_time": source_file.sensingtime.strftime(JSON_DATE_FORMAT),
"modification_time": source_file.mtime.strftime(JSON_DATE_FORMAT),
"modification_time": mtime.strftime(JSON_DATE_FORMAT),
"production_latency_delay": deltaTimePLD.days * 86400 + deltaTimePLD.seconds,
"local_update_delay": deltaTimeLUD.days * 86400 + deltaTimeLUD.seconds,
"file_volume": source_file.size,
"file_volume": size,
}
tookedSemaphore = False
jsonFile = None
......
......@@ -209,3 +209,11 @@ class AbstractProvider(object):
def setOption(self, optionDict):
self.__option = optionDict
def getmtime(self):
""" get method for mtime, see method above for more info """
return None
def getsize(self):
""" get method for size, see method above for more info """
return None
......@@ -13,6 +13,7 @@ import urllib.request
import urllib.error
import urllib.parse
import http.cookiejar
import datetime
from downloader.scheduler.com.sys.File import File, READ_BUFFER
......@@ -28,6 +29,7 @@ class HttpsSession(object):
self.filename = None
self.filesize = 0
self.option = option
self.mtime = None
socket.setdefaulttimeout(self.timeout)
......@@ -61,6 +63,24 @@ class HttpsSession(object):
if (self.opener is not None):
flow = self.opener.open(remotepath)
downloadfile = open(localpath, 'wb')
self.filesize = int(flow.info()['Content-Length'])
if 'Last-Modified' in flow.info():
try:
# RFC 822, updated by RFC 1123
self.mtime = datetime.datetime.strptime(flow.info()['Last-Modified'], '%a, %d %b %Y %H:%M:%S GMT')
except:
try:
# ANSI C's asctime() format
self.mtime = datetime.datetime.strptime(flow.info()['Last-Modified'], '%c')
except:
try:
# RFC 850, obsoleted by RFC 1036
self.mtime = datetime.datetime.strptime(flow.info()['Last-Modified'], '%A, %d-%b-%y %H:%M:%S GMT')
except:
self.mtime = None
else:
self.mtime = None
self.filesize = int(flow.info()['Content-Length'])
# check real filename
if 'Content-Disposition' in flow.info() \
......@@ -96,6 +116,12 @@ class HttpsSession(object):
def getFilename(self):
return(self.filename)
def getFilemtime(self):
return(self.mtime)
def getFilesize(self):
return(self.filesize)
class HttpsProvider(AbstractProvider):
"""
......@@ -112,6 +138,8 @@ class HttpsProvider(AbstractProvider):
self.__password = None
self.__session = None
self.__option = None
self.__file_mtime = None
self.__file_size = None
server = property(lambda self: self.__server, None, None, None)
username = property(lambda self: self.__username, None, None, None)
......@@ -182,6 +210,11 @@ class HttpsProvider(AbstractProvider):
# externe eventuellement.
target_file = session.getFilename()
self.__file_mtime = session.getFilemtime()
self.__file_size = session.getFilesize()
try:
os.rename(tmp_localpath, target_file)
except OSError as msg:
......@@ -220,3 +253,9 @@ class HttpsProvider(AbstractProvider):
def setOption(self, optionDict):
self.__option = optionDict
def getmtime(self):
return self.__file_mtime
def getsize(self):
return self.__file_size
netCDF4~=1.4.0
celery~=4.3.0
simplejson~=3.16.0
SQLAlchemy~=1.3.8
......@@ -11,3 +10,4 @@ elasticsearch~=6.4
urllib3<=1.24.3
eccodes-python~=0.9.7
numpy~=1.18
netCDF4~=1.5.0
......@@ -10,7 +10,7 @@ author-email = sherleda@ifremer.fr
license = GPLv3
# Python version
python_requires = ==3.7
requires-python = >=3.7 <3.8
# Installation -------------
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment