Commit 7ce7ecb0 authored by BODERE's avatar BODERE
Browse files

Merge branch 'hotfix_6.0.16' into 'master'

Hotfix 6.0.16

Closes #71, #77 et #79

See merge request downloader/downloader_daemon!48
parents 139a0dd3 64ceb4b9
......@@ -28,7 +28,6 @@ FILE_URL_SEPARATOR = " -|- "
Base = declarative_base()
class Constant(Base):
# SQLite table: constants_infos
__tablename__ = 'constants_infos'
......@@ -176,7 +175,8 @@ class SqliteBaseFilesInfos(object):
# Connect to SQL database
from sqlalchemy import create_engine
print("Open SQL db: %s", "sqlite:///" + self.__db_url)
self.__sql_engine = create_engine("sqlite:///" + self.__db_url, echo=False)
self.__sql_engine = create_engine("sqlite:///" + self.__db_url,
echo=False)
# Create all tables
Base.metadata.create_all(self.__sql_engine)
......@@ -186,6 +186,11 @@ class SqliteBaseFilesInfos(object):
self.__sql_session = sessionmaker(bind=self.__sql_engine)()
self.__sql_session.execute('PRAGMA foreign_keys=ON')
vacuum = self.__sql_session.execute('select * from pragma_auto_vacuum').fetchall()
if vacuum[0][0] == 0:
self.__sql_session.execute('PRAGMA auto_vacuum=1')
self.__sql_session.execute('vacuum')
version = self.__sql_session \
.query(ConfigEntry) \
.filter(ConfigEntry.key == "version") \
......
......@@ -529,7 +529,7 @@ class DCheck(object):
if len(args) > 0:
url = args[0]
#setConfigFromUrl(config, url)
setConfigFromUrl(config, url)
if options.debug_mode:
config.debug = True
......@@ -543,7 +543,7 @@ class DCheck(object):
if options.database:
config.database_path = options.database
else:
else: # ???
config.database_path = workingdir.getDatabasePathFromUrl(
url, prefix=config.protocol + '_', create_path=True)
......
......@@ -21,10 +21,7 @@ from downloader.scheduler.sc import LogUtil
from dchecktools.common.basefileinfos import FILE_URL_SEPARATOR
from downloader.scheduler.sc.GlobalConfig import GlobalConfig
VERSION = "1.0"
VERSION = "1.1"
LOGGER_DEFAULT_FILE_NAME = "reload_"
RELOAD_LOGGER_LEVEL = logging.INFO
......@@ -220,11 +217,17 @@ class ReLoader(object):
self.log.debug("Input '%s'...", filename)
if isSafe:
filename_template = '%/' + filename + '%'
if filename[0] in ('/', '*', '%'):
filename_template = filename + '%'
else:
filename_template = '%/' + filename + '%'
elif hasSeparator:
filename_template = filename + FILE_URL_SEPARATOR + '%'
else:
filename_template = '%/' + filename
if filename[0] in ('/', '*', '%'):
filename_template = filename
else:
filename_template = '%/' + filename
try:
list_Filepaths = self.__database.resetDownload(filename_template)
......
......@@ -409,7 +409,7 @@ class Download(threading.Thread):
# Ressources semaphore (parralel download)
self.__semaState = threading.Semaphore()
# 28/02/2018 PMT : ajout de verbose (ode debug)
# 28/02/2018 PMT : ajout de verbose (mode debug)
self.__verbose = False
self.__groupid = -1
......@@ -453,6 +453,9 @@ class Download(threading.Thread):
from .LogUtil import setupFileLogger
self.setFileLogName(path)
setupFileLogger(self._log, fileLogConfig, self.__logFile.getName())
self._log.setLevel(logging.DEBUG
if self.__verbose or self.__configuration.debug
else logging.INFO)
def setFileLogName(self, path):
fileName = [LOGGER_DEFAULT_FILE_NAME]
......@@ -883,7 +886,12 @@ class Download(threading.Thread):
else:
self.__listing_builder.setConfiguration(
self.__configuration.source.update)
self.__listing_builder.setModeDebug(True if self.__configuration.debug else self.__verbose)
if self.__configuration.debug:
self._log.setLevel(logging.DEBUG)
self.__listing_builder.setModeDebug(True)
else:
self.__listing_builder.setModeDebug(self.__verbose)
self._log.debug(" --> configure DataReader")
try:
......@@ -1177,7 +1185,7 @@ class Download(threading.Thread):
m = messages.MessageFactory().create({
'type': 'downloadMessage',
'level': 'WARNING',
'summary': '[%s] get listing anomaly' % (self.id),
'summary': '[%s] get listing anomaly : %s' % (self.id, str(e)),
'details': '%s' % (msg + ' : ' + str(e)),
'processing_start_date': self.__emm_start_date,
'processing_hostname': gethostname(),
......@@ -1372,7 +1380,7 @@ class Download(threading.Thread):
m = messages.MessageFactory().create({
'type': 'downloadMessage',
'level': 'WARNING',
'summary': '[%s] Download aborted : %s' % (self.id, os.path.basename(file)),
'summary': '[%s] Download policy violation for file %s' % (self.id, os.path.basename(file)),
'details': '%s' % (msg),
'reference_type': 'data',
'reference_name': os.path.basename(file),
......@@ -2306,7 +2314,7 @@ class Download(threading.Thread):
self._log.info(' => final directory %s:' %
(list_directories[index]))
links.sort()
#links.sort()
for link in links:
if isinstance(link, Folder):
linkName = link.getName()
......@@ -3572,6 +3580,8 @@ class Download(threading.Thread):
self._log.log(level, " " + "-" * 99)
self._log.log(level, " - Download '%s'" % self.id)
self._log.log(level, " test mode : '%s'" % self.test)
self._log.log(level, " log debug : '%s'" %
self.__configuration.debug)
self._log.log(level, " Download configuration file : '%s'" %
self.download_conf_file)
self._log.log(level, " data source : ")
......
......@@ -456,10 +456,10 @@ class SynchronisationLocalRemote(object):
filename = ".".join(splitFileName[:-1])
if filename not in listFilenameLocal:
self.__log.debug("not listFilenameLocal %s", filename)
self.__log.debug("%s not in local list", filename)
self.reportString(filepath, info=fileInfo)
elif self.modified:
self.__log.debug("Filename %s exist ", filename)
self.__log.debug("Filename %s exist in local list", filename)
_basedir, localmtime = listFilenameLocal[filename]
if mtime > localmtime:
self.__log.debug("mtime: %s > local mtime %s ",
......@@ -467,7 +467,7 @@ class SynchronisationLocalRemote(object):
self.reportString(filepath, info=fileInfo)
except AttributeError:
self.__log.warning("Cannot determine date filename %s",
self.__log.warning("Cannot determine date of filename %s",
filename)
self.setJobState([STATE_SUCCESS, ""])
......
......@@ -80,15 +80,14 @@ class WorkerUI(threading.Thread):
def run(self):
#self._log.info("workerUI is attached to the '%s' queue." % self.__queueName)
command = "celery"
argsList = [
# celery worker --config=(...).celery_config -l DEBUG -Q UI.hostname
'worker',
'--config',
CONFIG_WORKER,
"--workdir",
self.__globalConfig.getPath("dynamic_plugins"),
'worker',
"-l",
self.__debug,
"-Q",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment