Commit c41ced94 authored by BODERE's avatar BODERE
Browse files

Merge branch 'hotfix_6.0.3' into 'master'

Hotfix 6.0.3

Closes #49

See merge request downloader/downloader_daemon!25
parents d9de0cbc bd1712a4
...@@ -12,12 +12,15 @@ class EMMWriter(IMessageWriter): ...@@ -12,12 +12,15 @@ class EMMWriter(IMessageWriter):
def __init__(self, globalConfig): def __init__(self, globalConfig):
IMessageWriter.__init__(self) IMessageWriter.__init__(self)
self.projet_name = globalConfig["emm.project"]
self.message_type = globalConfig["emm.message_type"]
self.__log = logging.getLogger() self.__log = logging.getLogger()
self.__file_writer = None self.__file_writer = None
if "filesystem" in globalConfig["emm.targets"]: if "filesystem" in globalConfig["emm.targets"]:
path = globalConfig["emm.filesystem.path"] path = globalConfig["emm.filesystem.path"]
if os.path.isabs(path): if not os.path.isabs(path):
path = os.path.join(globalConfig["paths.workspace"], path) path = os.path.join(globalConfig["paths.workspace"], path)
self.__file_writer = EnhancedFileWriter( self.__file_writer = EnhancedFileWriter(
...@@ -69,6 +72,10 @@ class EMMWriter(IMessageWriter): ...@@ -69,6 +72,10 @@ class EMMWriter(IMessageWriter):
xml_declaration=True, xml_declaration=True,
is_send_toRabbit_mq=False): is_send_toRabbit_mq=False):
# overwrite project and type
message.project = self.projet_name
message.type = self.message_type
# Filesystem # Filesystem
if self.__file_writer is not None: if self.__file_writer is not None:
try: try:
...@@ -92,3 +99,6 @@ class EMMWriter(IMessageWriter): ...@@ -92,3 +99,6 @@ class EMMWriter(IMessageWriter):
# RabbitMQ # RabbitMQ
if self.__emm_rabbitmq is not None: if self.__emm_rabbitmq is not None:
self.__emm_rabbitmq.writeMessage(dataset_id, message) self.__emm_rabbitmq.writeMessage(dataset_id, message)
def setProjectName(self, project_name):
self.projet_name = project_name
\ No newline at end of file
...@@ -357,15 +357,15 @@ class File(object): ...@@ -357,15 +357,15 @@ class File(object):
else: else:
# create a explicite new file name for the uncompressed file # create a explicite new file name for the uncompressed file
unzipFile = File(self.getName() + ".un" + ct + "ed") unzipFile = File(self.getName() + ".un" + ct + "ed")
# Open the current compressed file
self.open("r")
# Open the new uncompressed file
if comptype == File.ZIP: if comptype == File.ZIP:
# Open the current compressed file
self.open("r")
filenames = self.__fd.namelist() filenames = self.__fd.namelist()
if len(filenames) > 1: if len(filenames) > 1:
raise Exception("ZipFile must only contains one file") raise Exception("ZipFile must only contains one file")
# Open the new uncompressed file
_filepath, name = os.path.split(filenames[0]) _filepath, name = os.path.split(filenames[0])
unzipFile = File(os.path.join(self.__path, name)) unzipFile = File(os.path.join(self.__path, name))
unzipFile.open("wb") unzipFile.open("wb")
...@@ -377,31 +377,36 @@ class File(object): ...@@ -377,31 +377,36 @@ class File(object):
self.close() self.close()
unzipFile.close() unzipFile.close()
raise raise
elif comptype == File.LZW:
data_uncompress = True
unzipFile.open("wb")
try:
data = self.__fd.read()
data_uncompress = unlzw(data)
unzipFile.write(data_uncompress)
except:
self.close()
unzipFile.close()
raise
else: else:
unzipFile.open("wb") # Open the current compressed file
# do the uncompression copy (with bufferization) self.open("rb")
buff = True if comptype == File.LZW:
try: data_uncompress = True
while buff: # Open the new uncompressed file
buff = self.read(READ_BUFFER) unzipFile.open("wb")
if buff: try:
unzipFile.write(buff) data = self.__fd.read()
except: data_uncompress = unlzw(data)
# Close the two files unzipFile.write(data_uncompress)
self.close() except:
unzipFile.close() self.close()
raise unzipFile.close()
raise
else:
# Open the new uncompressed file
unzipFile.open("wb")
# do the uncompression copy (with bufferization)
buff = True
try:
while buff:
buff = self.read(READ_BUFFER)
if buff:
unzipFile.write(buff)
except:
# Close the two files
self.close()
unzipFile.close()
raise
# Close the two files # Close the two files
self.close() self.close()
......
...@@ -623,6 +623,7 @@ class ConfigurationSettings(object): ...@@ -623,6 +623,7 @@ class ConfigurationSettings(object):
self.__maxNbOfConcurrentStreams = None self.__maxNbOfConcurrentStreams = None
self.__maxNbOfLinesInAutoListing = None self.__maxNbOfLinesInAutoListing = None
self.__monitoring = None self.__monitoring = None
self.__projectName = None
def Read(self): def Read(self):
...@@ -704,10 +705,15 @@ class ConfigurationSettings(object): ...@@ -704,10 +705,15 @@ class ConfigurationSettings(object):
self.__monitoring = True self.__monitoring = True
if self.__xr.haveSubNode(self.__node, 'monitoring'): if self.__xr.haveSubNode(self.__node, 'monitoring'):
monitoring = self.__xr.getSubNodeValue(logging.WARNING, self.__node, 'monitoring', "") monitoring = self.__xr.getSubNodeValue(logging.WARNING,
self.__node, 'monitoring', "")
if monitoring is not None and monitoring.lower() == 'false': if monitoring is not None and monitoring.lower() == 'false':
self.__monitoring = False self.__monitoring = False
if self.__xr.haveSubNode(self.__node, 'projet_name'):
self.__projectName = self.__xr.getSubNodeValue(logging.WARNING,
self.__node, 'projet_name')
return result return result
database_path = property( database_path = property(
...@@ -780,3 +786,9 @@ class ConfigurationSettings(object): ...@@ -780,3 +786,9 @@ class ConfigurationSettings(object):
None, None,
None, None,
"read only monitoring variable") "read only monitoring variable")
projectName = property(
lambda self: self.__projectName,
None,
None,
"read only projectName variable")
...@@ -640,6 +640,9 @@ class Download(threading.Thread): ...@@ -640,6 +640,9 @@ class Download(threading.Thread):
# ============================================================================================================== # ==============================================================================================================
self._log.debug(" --> apply downloader configuration : settings") self._log.debug(" --> apply downloader configuration : settings")
if self.__configuration.settings.projectName is not None:
self.__emm_writer.setProjectName(self.__configuration.settings.projectName)
if self.__configuration.settings.nbRetries is not None: if self.__configuration.settings.nbRetries is not None:
self.__misc_param_nb_retry = self.__configuration.settings.nbRetries self.__misc_param_nb_retry = self.__configuration.settings.nbRetries
...@@ -2575,7 +2578,7 @@ class Download(threading.Thread): ...@@ -2575,7 +2578,7 @@ class Download(threading.Thread):
else: else:
dl_state = STATE_DOWNLOAD_ERROR dl_state = STATE_DOWNLOAD_ERROR
self._log.debug("updateStateDownload of %s state : %d" % (filepath, dl_state)) self._log.debug("updateStateDownload of %s state IN : %d" % (filepath, dl_state))
# NetCDF Datareader Case: Searching for Sensingtime in Global NetCDF Attributes # NetCDF Datareader Case: Searching for Sensingtime in Global NetCDF Attributes
if source_file.sensingtime is None: if source_file.sensingtime is None:
source_file.sensingtime = self.__data_reader.getDate(downloaded_file.localFile.getName()) source_file.sensingtime = self.__data_reader.getDate(downloaded_file.localFile.getName())
...@@ -2586,7 +2589,7 @@ class Download(threading.Thread): ...@@ -2586,7 +2589,7 @@ class Download(threading.Thread):
self._log.debug("updateStateDownload of %s state None ?????????" % (filepath)) self._log.debug("updateStateDownload of %s state None ?????????" % (filepath))
msg_level = 'WARNING' msg_level = 'WARNING'
else: else:
self._log.debug("updateStateDownload of %s state : %d" % (filepath, db_state)) self._log.debug("updateStateDownload of %s state OUT: %d" % (filepath, db_state))
if msg_level == 'INFO': if msg_level == 'INFO':
m.level = messages.INFO m.level = messages.INFO
......
...@@ -30,6 +30,8 @@ class GlobalConfig(object): ...@@ -30,6 +30,8 @@ class GlobalConfig(object):
admins: !!list [] admins: !!list []
emm: emm:
project: CERSAT
message_type: download
targets: [] targets: []
filesystem: filesystem:
path: spools/message/ path: spools/message/
...@@ -66,8 +68,33 @@ class GlobalConfig(object): ...@@ -66,8 +68,33 @@ class GlobalConfig(object):
user: null user: null
password: null password: null
virtual_host: "/" virtual_host: "/"
queue_name: dl-jobs queue_name: dl-jobsif
routing_key: null routing_key: null
ui_worker:
queue: #{queue_name: { queue_arguments: {alias : alias_downloader_name}}}
name: UI.host.0
alias: alias_downloader_name
broker_url: #"amqp://user:password@host:5672/vhost"
host: localhost
port: 5672
user: null
password: null
virtual_host: "/"
broker_heartbeat: 30
imports: [downloader.worker.celery_tasks]
result_backend: amqp
result_expires: 300
task_serializer: json
result_serializer: json
timezone: Europe/Paris
enable_utc: True
result_compression: bzip2
worker_log_color: True
accept_content: [json]
auto_delete: True
expires: 15.0
exclusive: True
logs: logs:
default: default:
......
...@@ -232,17 +232,22 @@ class Scheduler(object): ...@@ -232,17 +232,22 @@ class Scheduler(object):
os.makedirs(path) os.makedirs(path)
# Setup file logging target # Setup file logging target
logFile = File(os.path.join(self.__globalConfig.getPath("process_log_archive"), "scheduler.log")) logFile = File(os.path.join(self.__globalConfig.getPath("process_log_archive"), "scheduler.log"))
if not logFile.acquireLock("%s%d" % (gethostname(), os.getpid())): lockfile = logFile.acquireLock("%s-%d %s" % (gethostname(), os.getpid(), self.__version))
if options.force:
self._log.info("Forcing the deletion of the lock file (%s)", logFile.getName() + ".lock") if not lockfile and options.force:
else: self._log.info("Forcing the deletion of the lock file (%s)", logFile.getName() + ".lock")
self._log.error("!" * 100) os.remove(logFile.getName() + ".lock")
self._log.error("A session of Scheduler for this DOWNLOADER_WORKSPACE, may be already run !") lockfile = logFile.acquireLock("%s-%d %s" % (gethostname(), os.getpid(), self.__version))
self._log.error("Check the lock file %s before restart the scheduler",
if not lockfile:
self._log.error("!" * 100)
self._log.error("A session of Scheduler for this DOWNLOADER_WORKSPACE, may be already run !")
self._log.error("Check the lock file %s before restart the scheduler",
logFile.getName() + ".lock") logFile.getName() + ".lock")
self._log.error("!" * 100) self._log.error("!" * 100)
sys.exit(EXIT_LOCKED) sys.exit(EXIT_LOCKED)
LogUtil.setupFileLogger( LogUtil.setupFileLogger(
self._log, self._log,
......
...@@ -41,8 +41,6 @@ class WorkerUI(threading.Thread): ...@@ -41,8 +41,6 @@ class WorkerUI(threading.Thread):
self._stopevent = threading.Event() self._stopevent = threading.Event()
configFile = self.__globalConfig.getPath("celery_cfg")
pluginFile = os.path.join(self.__globalConfig.getPath("dynamic_plugins"), CONFIG_WORKER + ".py") pluginFile = os.path.join(self.__globalConfig.getPath("dynamic_plugins"), CONFIG_WORKER + ".py")
plugin = open(pluginFile, "w") plugin = open(pluginFile, "w")
...@@ -50,6 +48,37 @@ class WorkerUI(threading.Thread): ...@@ -50,6 +48,37 @@ class WorkerUI(threading.Thread):
line = "downloader_configfile='" + os.path.abspath(self.__globalConfig.filePath) + "'\n" line = "downloader_configfile='" + os.path.abspath(self.__globalConfig.filePath) + "'\n"
plugin.write(line + "\n") plugin.write(line + "\n")
#Add queue
for key in self.__globalConfig["ui_worker"].keys():
if key == 'queue':
line = "task_queues={" + \
"'" + self.__globalConfig["ui_worker.queue.name"] + "': " +\
"{'queue_arguments': " +\
"{'alias': '" + self.__globalConfig["ui_worker.queue.alias"] + "'}," + \
"'vhost': '" + self.__globalConfig["ui_worker.broker_url.virtual_host"] + "'}" +\
"}"
self.__queueName = self.__globalConfig["ui_worker.queue.name"]
elif key == 'broker_url':
line = "broker_url="
line += "'amqp://"
line += str(self.__globalConfig["ui_worker.broker_url.user"]) + ":"
line += str(self.__globalConfig["ui_worker.broker_url.password"]) + "@"
line += str(self.__globalConfig["ui_worker.broker_url.host"]) + ":"
line += str(self.__globalConfig["ui_worker.broker_url.port"]) + "/"
line += str(self.__globalConfig["ui_worker.broker_url.virtual_host"]) + "'"
else:
entry = "ui_worker." + key
if isinstance(self.__globalConfig[entry], (str, bool)):
value = "'" + str(self.__globalConfig[entry]) + "'"
else:
value = str(self.__globalConfig[entry])
line = key + "=" + value
plugin.write(line + "\n")
"""
configFile = self.__globalConfig.getPath("celery_cfg")
# search (and replace default queue name) UI queue name in configuration file # search (and replace default queue name) UI queue name in configuration file
with open(configFile, 'r') as f: with open(configFile, 'r') as f:
config = simplejson.loads(f.read()) config = simplejson.loads(f.read())
...@@ -68,7 +97,7 @@ class WorkerUI(threading.Thread): ...@@ -68,7 +97,7 @@ class WorkerUI(threading.Thread):
self.__queueName = queue self.__queueName = queue
self._log.debug("config queue OK") self._log.debug("config queue OK")
break break
"""
plugin.close() plugin.close()
def run(self): def run(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment