Commit 1a041e5a authored by pm22d12's avatar pm22d12
Browse files

Adding configuration loading errors

parent 5e87de9b
......@@ -10,7 +10,7 @@ urlpatterns = [
url(r'^nbDesactivatedFile$', views.getDesactivatedNbFile, name='nbDeactivatedFile'),
url(r'^nbTestFile$', views.getTestNbFile, name='nbTestFile'),
url(r'^help$', views.help, name='help'),
url(r'^filterdownloads/(?P<pattern>\S+)$', views.filter_downloads, name='filterdownloads'),
#url(r'^filterdownloads/(?P<pattern>\S+)$', views.filter_downloads, name='filterdownloads'),
url(r'^createdownload/(?P<download_id>\S+)$', views.create_and_edit_download, name='createdownload'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/define_test_file$', views.define_test_file, name='define_test_file'),
......@@ -33,7 +33,7 @@ urlpatterns = [
url(r'^nbfile/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.nb_file_synchro, name='nb_file_synchro'),
url(r'^date/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.date_synchro, name='date_synchro'),
url(r'^checksynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.check_synchro, name='checksynchro'),
url(r'^runsynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.run_synchro, name='runsynchro'),
#url(r'^runsynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.run_synchro, name='runsynchro'),
url(r'^(?P<xml_filename>\S+)/save$', views.save, name='save'),
......
from lxml import etree, objectify
from lxml.etree import XMLSyntaxError
import time
import os
from lxml import etree, objectify
from lxml.etree import XMLSyntaxError
from django.utils.safestring import mark_safe
import downloader_django.common.constants
from downloader_django.common.system import tasks
......@@ -61,6 +61,58 @@ def desactivate_download(queue, filename):
return False
def get_downloads_with_error(queue):
mapFile = {}
mapFile["ON"] = []
mapFile["OFF"] = []
mapFile["TEST"] = []
map_file_id = {}
map_file_id_test = {}
text_error = ""
for filename in tasks.get_configuration_files(queue):
mapFileName = {}
map = ""
error = ""
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
if fileExtension == ".xml" and filesplit[-1] == "download":
map = "ON"
elif fileExtension == ".xml" and filesplit[-1] == "test" and filesplit[-2] == "download":
map = "TEST"
elif fileExtension in (".OFF", ".BAD") and filesplit[-1] == "xml" and filesplit[-2] == "download":
map = "OFF"
elif filename[0:1] != '.':
error = f"<b>{filename}</b> is not a configuration file"
else:
continue
if error == "":
try:
file_data = get_xml_data(queue, filename)
mapData = []
mapData.append(file_data)
mapFileName[filename] = mapData
mapFile[map].append(mapFileName)
if map == "ON":
map_file_id[filename] = file_data.attrib['id']
if map == "TEST":
map_file_id_test[filename] = file_data.attrib['id']
except XMLSyntaxError as e:
error = f"<b>{filename}</b> => {str(e)}"
if error != "":
if text_error != "":
text_error += "<br>"
text_error += error
return mapFile, map_file_id, map_file_id_test, mark_safe(text_error)
def get_downloads(queue):
mapFile = {}
......@@ -200,8 +252,8 @@ def copy_as_file(queue, src, dst, download_id_dst):
# replace tag post_processing by post-processing
if len(file_data.data_destination.findall("post_processing")) > 0:
file_data.data_destination.findall("post_processing")[
0].tag = "post-processing"
file_data.data_destination.findall("post_processing")[0].tag =\
"post-processing"
objectify.deannotate(file_data, cleanup_namespaces=True)
xml_string = etree.tostring(file_data, pretty_print=True)
......@@ -235,8 +287,8 @@ def set_download_config(queue, xml_file_name, submitted_data):
# replace tag post_processing by post-processing
if len(file_data.data_destination.findall("post_processing")) > 0:
file_data.data_destination.findall("post_processing")[
0].tag = "post-processing"
file_data.data_destination.findall("post_processing")[0].tag =\
"post-processing"
objectify.deannotate(file_data, cleanup_namespaces=True)
xmlString = etree.tostring(file_data, pretty_print=True)
......
......@@ -62,6 +62,7 @@ class Synchro(Process):
name = download_id + "_" + current_time.strftime('%Y%m%d_%H%M%S')
synchro_output_filename = "%s.synchro" % (name)
self.download_id = download_id
self.modeDownload = modeDownload
self.queue = queue
self.celery_queue = celery_queue
......@@ -78,10 +79,13 @@ class Synchro(Process):
def run(self):
is_success = run_synchro(self.celery_queue,
self.configuration_filename,
self.listing_filename,
self.job_filename,
self.synchro_subpath)
is_success = run_synchro(
self.download_id,
self.celery_queue,
self.configuration_filename,
self.listing_filename,
self.job_filename,
self.synchro_subpath
)
self.queue.put(is_success)
......@@ -105,13 +105,20 @@ def listdownload(request):
queue = request.session['queue'] if 'queue' in request.session else ""
if queue == '':
file_map, map_file_id, map_file_id_test = {}, {}, {}
text_error = ""
else:
file_map, map_file_id, map_file_id_test = downloaderXml.get_downloads(queue)
file_map, map_file_id, map_file_id_test, text_error = downloaderXml.get_downloads_with_error(queue)
synchro.add_synchro_status(queue, file_map)
return render(
request,
'adminview/activateddownload.html',
{'file_map': file_map, 'map_file_id': map_file_id, 'map_file_id_test': map_file_id_test},
{
'file_map': file_map,
'map_file_id': map_file_id,
'map_file_id_test': map_file_id_test,
'error': text_error,
'file_name': '',
},
)
......@@ -288,11 +295,10 @@ def display_test_log(request, download_id):
def check_synchro(request, xml_filename, download_id):
error = ""
q = Queue()
synchro_process = Synchro(request.session['queue'], q, xml_filename, download_id, False)
queue = Queue()
synchro_process = Synchro(request.session['queue'], queue, xml_filename, download_id, False)
synchro_state = synchro_process.check_status_synchro()
if synchro_state == "stop":
synchro_process.start()
time.sleep(2)
......@@ -305,6 +311,7 @@ def check_synchro(request, xml_filename, download_id):
return HttpResponse(error)
"""
def run_synchro(request, xml_filename, download_id):
error = ""
q = Queue()
......@@ -315,6 +322,7 @@ def run_synchro(request, xml_filename, download_id):
if not synchro_process.queue.get():
error = "synchro_not_started"
return HttpResponse(error)
"""
def status_synchro(request, xml_filename, download_id):
......@@ -476,15 +484,23 @@ def new_queue(request):
queue = request.POST['new_queue']
request.session['queue'] = queue
if queue == '':
file_map, map_file_id, map_file_id_test = {}, {}, {}
text_error = ""
else:
file_map, map_file_id, map_file_id_test = downloaderXml.get_downloads(queue)
file_map, map_file_id, map_file_id_test, text_error = downloaderXml.get_downloads_with_error(queue)
return render(
request,
'adminview/activateddownload.html',
{'file_map': file_map, 'map_file_id': map_file_id, 'map_file_id_test': map_file_id_test},
{
'file_map': file_map,
'map_file_id': map_file_id,
'map_file_id_test': map_file_id_test,
'error': text_error,
'file_name': ''
},
)
def help(request):
......
......@@ -10,7 +10,6 @@ DOWNLOADER_QUEUE = ''
def archive_download(queue, filename, work_id, suffix):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'archive_download',
......@@ -18,16 +17,14 @@ def archive_download(queue, filename, work_id, suffix):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
# -------------------------------------------------------------
def get_configuration_files(queue):
config = []
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'get_configuration_files',
......@@ -35,14 +32,12 @@ def get_configuration_files(queue):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
config = result.info
return config
return result.info
except TimeoutError:
return config
return []
def read_configuration_file(queue, filename):
data = None
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'read_configuration_file',
......@@ -50,14 +45,12 @@ def read_configuration_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
data = result.info
return data
return result.info
except TimeoutError:
return data
return None
def rename_configuration_file(queue, filename, fileNew):
isSuccess = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'rename_configuration_file',
......@@ -65,14 +58,12 @@ def rename_configuration_file(queue, filename, fileNew):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return isSuccess
return False
def copy_configuration_file(queue, filename, filepath):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'copy_configuration_file',
......@@ -80,14 +71,12 @@ def copy_configuration_file(queue, filename, filepath):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
def remove_configuration_file(queue, filename):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'remove_file',
......@@ -95,14 +84,12 @@ def remove_configuration_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
def write_configuration_file(queue, filename, content):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'write_configuration_file',
......@@ -110,14 +97,12 @@ def write_configuration_file(queue, filename, content):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
def exists_configuration_file(queue, filename):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'exists_configuration_file',
......@@ -125,16 +110,15 @@ def exists_configuration_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
# -------------------------------------------------------------
return False
# -------------------------------------------------------------
# Testing
# -------------------------------------------------------------
def read_test_state_file(queue, filename):
data = None
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'read_test_state_file',
......@@ -142,16 +126,15 @@ def read_test_state_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
data = result.info
return data
return result.info
except TimeoutError:
return data
# -------------------------------------------------------------
return None
# -------------------------------------------------------------
# Reloading
# -------------------------------------------------------------
def run_reload(queue, xml_filename, data, data_filename):
is_success = -2
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'run_reload',
......@@ -159,16 +142,15 @@ def run_reload(queue, xml_filename, data, data_filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
# -------------------------------------------------------------
return -2
# -------------------------------------------------------------
# Synchronization
# -------------------------------------------------------------
def check_status_synchro(queue, synchro_job_file):
state = None
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'check_status_synchro',
......@@ -176,31 +158,26 @@ def check_status_synchro(queue, synchro_job_file):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
state = result.info
return state
return result.info
except TimeoutError:
return state
return None
def run_synchro(queue, configuration_filename, listing_filename, synchro_job_file, synchro_subpath):
is_success = False
def run_synchro(download_id, queue, configuration_filename, listing_filename, synchro_job_file, synchro_subpath):
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'run_synchro',
(configuration_filename, listing_filename,
(download_id, configuration_filename, listing_filename,
synchro_job_file, synchro_subpath),
queue=queue
)
result.get()
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
def read_synchro_state_file(queue, filename):
data = None
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'read_synchro_state_file',
......@@ -208,14 +185,12 @@ def read_synchro_state_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
data = result.info
return data
return result.info
except TimeoutError:
return data
return None
def read_log_file(queue, filename):
data = None
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'read_log_file',
......@@ -223,31 +198,25 @@ def read_log_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
data = result.info
return data
return result.info
except TimeoutError:
return data
# -------------------------------------------------------------
return None
def read_listing_file(queue, full_filename):
data = None
def copy_listing_file(queue, filepath, work_id):
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'read_listing_file',
(full_filename,),
DOWNLOADER_TASKS + 'copy_listing_file',
(filepath, work_id),
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
data = result.info
return data
return result.info
except TimeoutError:
return data
return False
def write_manual_listing(queue, filename, work_id, content):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'write_manual_listing',
......@@ -255,59 +224,54 @@ def write_manual_listing(queue, filename, work_id, content):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return result.info
except TimeoutError:
return is_success
return False
def read_manual_listing(queue, filename, work_id):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'read_manual_listing',
(filename, work_id),
queue=queue
)
data = result.info
return data
result.get(timeout=RABBITMQ_TIMEOUT)
return result.info
except TimeoutError:
return data
return None
def copy_listing_file(queue, filepath, work_id):
is_success = False
def read_listing_file(queue, full_filename):
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'copy_listing_file',
(filepath, work_id),
DOWNLOADER_TASKS + 'read_listing_file',
(full_filename,),
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
return is_success
return result.info
except TimeoutError:
return is_success
return None
# --------------------------------------------------------------------------------------
def rename_a_file(queue, filename, fileNew):
isSuccess = False
def rename_a_file(queue, filename, newfile):
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'rename_file',
(filename, fileNew),
(filename, newfile),
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return isSuccess
return False
def copy_file(queue, filename, filepath):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'copy_file',
......@@ -315,14 +279,12 @@ def copy_file(queue, filename, filepath):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
def remove_file(queue, filename):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'remove_file',
......@@ -330,14 +292,12 @@ def remove_file(queue, filename):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success
return result.info
except TimeoutError:
return is_success
return False
def write_content_file(queue, filename, content):
is_success = False
try:
result = settings.CELERY.send_task(
DOWNLOADER_TASKS + 'create_file',
......@@ -345,7 +305,6 @@ def write_content_file(queue, filename, content):
queue=queue
)
result.get(timeout=RABBITMQ_TIMEOUT)
is_success = result.info
return is_success