Commit 6d3e9ff4 authored by MAISSIAT's avatar MAISSIAT
Browse files

Fix #77 : Log EMM - add the error message, if exists, for warnings

Fix #71 : Add optional spool path for downloaded files copy without post-processing
parent 139a0dd3
......@@ -529,7 +529,7 @@ class DCheck(object):
if len(args) > 0:
url = args[0]
#setConfigFromUrl(config, url)
setConfigFromUrl(config, url)
if options.debug_mode:
config.debug = True
......@@ -543,7 +543,7 @@ class DCheck(object):
if options.database:
config.database_path = options.database
else:
else: # ???
config.database_path = workingdir.getDatabasePathFromUrl(
url, prefix=config.protocol + '_', create_path=True)
......
......@@ -21,10 +21,7 @@ from downloader.scheduler.sc import LogUtil
from dchecktools.common.basefileinfos import FILE_URL_SEPARATOR
from downloader.scheduler.sc.GlobalConfig import GlobalConfig
VERSION = "1.0"
VERSION = "1.1"
LOGGER_DEFAULT_FILE_NAME = "reload_"
RELOAD_LOGGER_LEVEL = logging.INFO
......@@ -220,11 +217,17 @@ class ReLoader(object):
self.log.debug("Input '%s'...", filename)
if isSafe:
filename_template = '%/' + filename + '%'
if filename[0] in ('/', '*', '%'):
filename_template = filename + '%'
else:
filename_template = '%/' + filename + '%'
elif hasSeparator:
filename_template = filename + FILE_URL_SEPARATOR + '%'
else:
filename_template = '%/' + filename
if filename[0] in ('/', '*', '%'):
filename_template = filename
else:
filename_template = '%/' + filename
try:
list_Filepaths = self.__database.resetDownload(filename_template)
......
......@@ -409,7 +409,7 @@ class Download(threading.Thread):
# Ressources semaphore (parralel download)
self.__semaState = threading.Semaphore()
# 28/02/2018 PMT : ajout de verbose (ode debug)
# 28/02/2018 PMT : ajout de verbose (mode debug)
self.__verbose = False
self.__groupid = -1
......@@ -453,6 +453,9 @@ class Download(threading.Thread):
from .LogUtil import setupFileLogger
self.setFileLogName(path)
setupFileLogger(self._log, fileLogConfig, self.__logFile.getName())
self._log.setLevel(logging.DEBUG
if self.__verbose or self.__configuration.debug
else logging.INFO)
def setFileLogName(self, path):
fileName = [LOGGER_DEFAULT_FILE_NAME]
......@@ -883,7 +886,12 @@ class Download(threading.Thread):
else:
self.__listing_builder.setConfiguration(
self.__configuration.source.update)
self.__listing_builder.setModeDebug(True if self.__configuration.debug else self.__verbose)
if self.__configuration.debug:
self._log.setLevel(logging.DEBUG)
self.__listing_builder.setModeDebug(True)
else:
self.__listing_builder.setModeDebug(self.__verbose)
self._log.debug(" --> configure DataReader")
try:
......@@ -1177,7 +1185,7 @@ class Download(threading.Thread):
m = messages.MessageFactory().create({
'type': 'downloadMessage',
'level': 'WARNING',
'summary': '[%s] get listing anomaly' % (self.id),
'summary': '[%s] get listing anomaly : %s' % (self.id, str(e)),
'details': '%s' % (msg + ' : ' + str(e)),
'processing_start_date': self.__emm_start_date,
'processing_hostname': gethostname(),
......@@ -1372,7 +1380,7 @@ class Download(threading.Thread):
m = messages.MessageFactory().create({
'type': 'downloadMessage',
'level': 'WARNING',
'summary': '[%s] Download aborted : %s' % (self.id, os.path.basename(file)),
'summary': '[%s] Download policy violation for file %s' % (self.id, os.path.basename(file)),
'details': '%s' % (msg),
'reference_type': 'data',
'reference_name': os.path.basename(file),
......@@ -2306,7 +2314,7 @@ class Download(threading.Thread):
self._log.info(' => final directory %s:' %
(list_directories[index]))
links.sort()
#links.sort()
for link in links:
if isinstance(link, Folder):
linkName = link.getName()
......@@ -3572,6 +3580,8 @@ class Download(threading.Thread):
self._log.log(level, " " + "-" * 99)
self._log.log(level, " - Download '%s'" % self.id)
self._log.log(level, " test mode : '%s'" % self.test)
self._log.log(level, " log debug : '%s'" %
self.__configuration.debug)
self._log.log(level, " Download configuration file : '%s'" %
self.download_conf_file)
self._log.log(level, " data source : ")
......
......@@ -456,10 +456,10 @@ class SynchronisationLocalRemote(object):
filename = ".".join(splitFileName[:-1])
if filename not in listFilenameLocal:
self.__log.debug("not listFilenameLocal %s", filename)
self.__log.debug("%s not in local list", filename)
self.reportString(filepath, info=fileInfo)
elif self.modified:
self.__log.debug("Filename %s exist ", filename)
self.__log.debug("Filename %s exist in local list", filename)
_basedir, localmtime = listFilenameLocal[filename]
if mtime > localmtime:
self.__log.debug("mtime: %s > local mtime %s ",
......@@ -467,7 +467,7 @@ class SynchronisationLocalRemote(object):
self.reportString(filepath, info=fileInfo)
except AttributeError:
self.__log.warning("Cannot determine date filename %s",
self.__log.warning("Cannot determine date of filename %s",
filename)
self.setJobState([STATE_SUCCESS, ""])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment