Commit 63dd133d authored by pm22d12's avatar pm22d12
Browse files

Fix #1: View & add -> Add button not binded

Fix #2: Broken functions
parent 9f2e8760
......@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="jdk" jdkName="Python 3.7 (downloader_ui)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
......
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.5" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (downloader_ui)" project-jdk-type="Python SDK" />
<component name="PyCharmProfessionalAdvertiser">
<option name="shown" value="true" />
</component>
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />
</component>
</project>
\ No newline at end of file
{ "CELERY_QUEUES": {"UI.ananda.1": { "vhost": "downloader"},
"UI.ananda.2": { "vhost": "downloader"},
"UI.vcerdownload1-test.1": { "vhost": "downloader"}
}
, "BROKER_URL": "amqp://zenika_training:zenika_training@vrabbitmq1-val.ifremer.fr:5672/downloader"
, "BROKER_HEARTBEAT":30
, "CELERY_IMPORTS": []
, "CELERY_RESULT_BACKEND": "amqp"
, "CELERY_TASK_RESULT_EXPIRES": 900
, "CELERY_TASK_SERIALIZER": "json"
, "CELERY_RESULT_SERIALIZER": "json"
, "CELERY_TIMEZONE": "Europe/Paris"
, "CELERY_ENABLE_UTC": "True"
, "CELERY_MESSAGE_COMPRESSION": "bzip2"
, "CELERYD_LOG_COLOR": "False"
, "CELERY_ACCEPT_CONTENT": ["json"]
}
\ No newline at end of file
{ "broker_url": "amqp://zenika_training:zenika_training@vrabbitmq1-val.ifremer.fr:5672/downloader",
"broker_heartbeat": 30,
"imports": [],
"result_backend": "amqp",
"result_expires": 300,
"task_serializer": "json",
"result_serializer": "json",
"timezone": "Europe/Paris",
"enable_utc": "True",
"result_compression": "bzip2",
"worker_log_color": "False",
"accept_content": ["json"],
"auto_delete": "True",
"expires": 15.0,
"exclusive": "True"
}
......@@ -5,3 +5,4 @@ location /dev/downloader {
client_max_body_size 200M;
}
......@@ -9,23 +9,7 @@ import simplejson
# Get settings from environment variables
# -----------------------------------------------------------------------------
"""
if 'DOWNLOADER_WORKSPACE' not in os.environ:
raise Exception('Missing DOWNLOADER_WORKSPACE environment variable.')
if not os.path.exists(os.environ['DOWNLOADER_WORKSPACE']):
raise Exception('Downloader worspace directory %s not found.' % os.environ['DOWNLOADER_WORKSPACE'])
WORKSPACE = os.environ.get('DOWNLOADER_WORKSPACE', '')
if 'DOWNLOADER_APPDATA' not in os.environ:
raise Exception('Missing DOWNLOADER_APPDATA environment variable.')
if not os.path.exists(os.environ['DOWNLOADER_APPDATA']):
raise Exception('Downloader application data directory %s not found.' % os.environ['DOWNLOADER_APPDATA'])
APPDATA = os.environ.get('DOWNLOADER_APPDATA', '')
if 'FELYX_CELERY_CFG' not in os.environ:
raise Exception('Missing FELYX_CELERY_CFG environment variable.')
if not os.path.exists(os.environ['FELYX_CELERY_CFG']):
raise Exception('Felyx Celery configuration file %s not found.' % os.environ['FELYX_CELERY_CFG'])
"""
if 'CELERY_CFG' not in os.environ:
raise Exception('Missing CELERY_CFG environment variable.')
if not os.path.exists(os.environ['CELERY_CFG']):
......@@ -58,7 +42,7 @@ DATABASES = {
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []
ALLOWED_HOSTS = ['vcerdownload1-test.ifremer.fr']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
......@@ -129,7 +113,16 @@ SECRET_KEY = 't25jsr7eh9!it09qhw4!^%zg)0&)l3#g^!pyj!#0w+^@2*3x9h'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
#'django.template.context_processors.debug',
'django.template.context_processors.request'
# ...
],
#'builtins': ["debugtools.templatetags.debugtools_tags"],
},
},
]
......@@ -170,6 +163,8 @@ INSTALLED_APPS = (
#'downloader_django.apps.monitor',
)
SESSION_ENGINE = "django.contrib.sessions.backends.db"
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
......@@ -203,26 +198,6 @@ LOGGING = {
# with open(infile_path, 'r') as infile:
# paths = objectify.parse(infile).getroot()
# CONF_PATH= os.path.join(WORKSPACE if paths.downloads_config.get('location') == "workspace" else APPDATA,
# str(paths.downloads_config))
# WORKSPACE_DOWNLOAD= os.path.join(WORKSPACE if paths.work.get('location') == "workspace" else APPDATA,
# str(paths.work))
# LOG_PATH = os.path.join(WORKSPACE if paths.process_log_archive.get('location') == "workspace" else APPDATA,
# str(paths.process_log_archive))
# CACHE_JOB_PATH = os.path.join(WORKSPACE if paths.jobfile.get('location') == "workspace" else APPDATA,
# str(paths.jobfile))
# CACHE_PROVIDER_PATH = os.path.join(WORKSPACE if paths.providerfile.get('location') == "workspace" else APPDATA,
# str(paths.providerfile))
#BIN_PATH= str(paths.exec_path)
#CACHE_SYNCHRO_PATH = os.path.join(WORKSPACE, 'cache/synchrofile/')
#infile_path = os.path.join(APPDATA, 'controller/sc_global.xml')
# with open(infile_path, 'r') as infile:
# global_config = objectify.parse(infile).getroot()
#LOCK_FILE_EXTENSION = global_config.lockfile_extension
# This loads the celery configuration from FELYX_CELERY_CFG file
with open(os.environ['CELERY_CFG'], 'rb') as f:
CELERY_CONF = simplejson.loads(f.read())
......
......@@ -21,18 +21,22 @@ urlpatterns = [
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/copy_as_file', views.copy_as_file, name='copy_as_file'),
url(r'^change_queue$', views.new_queue, name='change_queue'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/reload_from_file', views.reload_from_file, name='reload_from_file'),
url(r'^synchro/listing/download/(?P<download_id>\S+)', views.move_synchro_listing_to_download, name='move_synchro_listing_to_download'),
url(r'^synchro/listing/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.display_synchro_listing, name='display_synchro_listing'),
url(r'^synchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.status_synchro, name='status_synchro'),
url(r'^test/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.status_test, name='status_test'),
url(r'^nbfile/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.nb_file_synchro, name='nb_file_synchro'),
url(r'^date/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.date_synchro, name='date_synchro'),
url(r'^checksynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.check_synchro, name='checksynchro'),
url(r'^runsynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.run_synchro, name='runsynchro'),
url(r'^(?P<xml_filename>\S+)/save$', views.save, name='save'),
url(r'^(?P<xml_filename>\S+)/activate$', views.activate, name='activate'),
url(r'^(?P<xml_filename>\S+)/desactivate$', views.desactivate, name='deactivate'),
url(r'^(?P<xml_filename>\S+)/delete$', views.delete, name='delete'),
......@@ -40,3 +44,5 @@ urlpatterns = [
url(r'^(?P<xml_filename>\S+)$', views.editdownload, name='edit'),
]
......@@ -7,7 +7,7 @@ import downloader_django.common.constants
from downloader_django.common.system import tasks
def activate_download(filename):
def activate_download(queue, filename):
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
......@@ -16,57 +16,52 @@ def activate_download(filename):
else:
newname = os.path.splitext(filename)[0]
if tasks.exists_configuration_file(filename):
is_success = tasks.rename_configuration_file(filename, newname)
if tasks.exists_configuration_file(queue, filename):
is_success = tasks.rename_configuration_file(queue, filename, newname)
return is_success
else:
return False
def delete_download(filename):
def delete_download(queue, filename):
is_success = tasks.remove_configuration_file(filename)
is_success = tasks.remove_configuration_file(queue, filename)
return is_success
def mode_test(filename):
def mode_test(queue, filename):
name, fileExtension = os.path.splitext(filename)
if fileExtension == ".xml":
newname = name + ".test" + fileExtension
elif fileExtension == ".OFF":
elif fileExtension in (".OFF", ".BAD"):
filesplit = name.split(".")
newname = (".").join(filesplit[:-1]) + ".test." + filesplit[-1]
if tasks.exists_configuration_file(filename):
return tasks.rename_configuration_file(filename, newname)
if tasks.exists_configuration_file(queue, filename):
return tasks.rename_configuration_file(queue, filename, newname)
else:
return False
def desactivate_download(filename):
def desactivate_download(queue, filename):
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
if tasks.exists_configuration_file(filename):
if tasks.exists_configuration_file(queue, filename):
if fileExtension == ".xml" and filesplit[-1] == "test":
newname = (".").join(filesplit[:-1]) + '.xml.OFF'
else:
newname = filename + ".OFF"
return tasks.rename_configuration_file(filename, newname)
return tasks.rename_configuration_file(queue, filename, newname)
else:
return False
"""
List files available
"""
def get_downloads():
def get_downloads(queue):
mapFile = {}
mapFile["ON"] = []
......@@ -75,13 +70,13 @@ def get_downloads():
map_file_id = {}
map_file_id_test = {}
for filename in tasks.get_configuration_files():
for filename in tasks.get_configuration_files(queue):
mapFileName = {}
if True:
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
try:
file_data = get_xml_data(filename)
file_data = get_xml_data(queue, filename)
mapData = []
mapData.append(file_data)
mapFileName[filename] = mapData
......@@ -92,35 +87,32 @@ def get_downloads():
elif fileExtension == ".xml" and filesplit[-1] == "test" and filesplit[-2] == "download":
mapFile["TEST"].append(mapFileName)
map_file_id_test[filename] = file_data.attrib['id']
elif fileExtension == ".OFF" and filesplit[-1] == "xml" and filesplit[-2] == "download":
elif fileExtension in (".OFF", ".BAD") and filesplit[-1] == "xml" and filesplit[-2] == "download":
mapFile["OFF"].append(mapFileName)
except XMLSyntaxError:
pass
# TODO: Commented for py3 migration: In py3 you can't sort a list of dicts, which totally makes sense
# mapFile["ON"].sort()
# mapFile["TEST"].sort()
# mapFile["OFF"].sort()
return mapFile, map_file_id, map_file_id_test
def filter_downloads(pattern):
def filter_downloads(queue, pattern):
map_downloads = {}
for filename in tasks.get_configuration_files():
if os.path.isfile(os.path.join(path, filename)):
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
for filename in tasks.get_configuration_files(queue):
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
if (fileExtension == ".xml" and filesplit[-1] == "download") \
or (fileExtension == ".xml" and filesplit[-1] == "test" and filesplit[-2] == "download") \
or (fileExtension in (".OFF", ".BAD") and filesplit[-1] == "xml" and filesplit[-2] == "download"):
try:
file_data = get_xml_data(queue, filename)
download_id = file_data.attrib['id']
if download_id.startswith(pattern):
map_downloads[download_id] = filename
except XMLSyntaxError:
pass
if (fileExtension == ".xml" and filesplit[-1] == "download") \
or (fileExtension == ".xml" and filesplit[-1] == "test" and filesplit[-2] == "download") \
or (fileExtension == ".OFF" and filesplit[-1] == "xml" and filesplit[-2] == "download"):
try:
file_data = get_xml_data(filename)
download_id = file_data.attrib['id']
if download_id.startswith(pattern):
map_downloads[download_id] = filename
except XMLSyntaxError:
pass
return map_downloads
......@@ -131,9 +123,9 @@ def get_schema():
return schema
def get_xml_data(xml_file_name):
def get_xml_data(queue, xml_file_name):
data = tasks.read_configuration_file(xml_file_name)
data = tasks.read_configuration_file(queue, xml_file_name)
schema = get_schema()
parser = objectify.makeparser(strip_cdata=False, schema=schema)
......@@ -146,7 +138,7 @@ def get_xml_data(xml_file_name):
return file_data
def create_download(xml_file_name):
def create_download(queue, xml_file_name):
default_id = xml_file_name.split(".download.")[0]
file_data = objectify.Element("download_config")
......@@ -193,15 +185,15 @@ def create_download(xml_file_name):
xml_string = etree.tostring(file_data, pretty_print=True)
valid_xml(xml_string)
is_success = save_download(xml_file_name, xml_string)
is_success = save_download(queue, xml_file_name, xml_string)
return is_success
def copy_as_file(src, dst, download_id_dst):
def copy_as_file(queue, src, dst, download_id_dst):
is_success = tasks.copy_configuration_file(src, dst)
is_success = tasks.copy_configuration_file(queue, src, dst)
if is_success:
while not tasks.exists_configuration_file(dst):
while not tasks.exists_configuration_file(queue, dst):
time.sleep(0.1)
file_data = get_xml_data(dst)
file_data.attrib['id'] = download_id_dst
......@@ -215,7 +207,7 @@ def copy_as_file(src, dst, download_id_dst):
xml_string = etree.tostring(file_data, pretty_print=True)
valid_xml(xml_string)
is_success = save_download(dst, xml_string)
is_success = save_download(queue, dst, xml_string)
return is_success
......@@ -225,25 +217,22 @@ def copy_as_file(src, dst, download_id_dst):
"""
def save_download(xml_file_name, xml_string):
def save_download(queue, xml_file_name, xml_string):
is_success = False
is_success = tasks.write_configuration_file(xml_file_name, xml_string)
is_success = tasks.write_configuration_file(queue, xml_file_name, xml_string)
return is_success
def set_download_config(xml_file_name, submitted_data):
file_data = get_xml_data(xml_file_name)
def set_download_config(queue, xml_file_name, submitted_data):
file_data = get_xml_data(queue, xml_file_name)
# id # TODO verifier unicite identifiant
if 'id' in submitted_data:
idDownload = submitted_data['id']
save_data_source(file_data, submitted_data)
save_data_destination(file_data, submitted_data)
save_download_settings(file_data, submitted_data)
print(file_data)
# replace tag post_processing by post-processing
if len(file_data.data_destination.findall("post_processing")) > 0:
file_data.data_destination.findall("post_processing")[
......
import os
import datetime
from multiprocessing import Process, Queue
from multiprocessing import Process
from downloader_django.common.constants import SYNCHRO_HISTORY
......@@ -10,7 +9,7 @@ from downloader_django.common.system.tasks import run_synchro, read_synchro_stat
copy_listing_file, check_status_synchro
def add_synchro_status(file_map):
def add_synchro_status(queue, file_map):
list_map_filename = file_map["ON"]
for map_filename in list_map_filename:
......@@ -19,7 +18,7 @@ def add_synchro_status(file_map):
mapFileName = values[0]
id = mapFileName.attrib['id']
map_synchro_status = read_synchro_status(id)
map_synchro_status = read_synchro_status(queue, id)
values.append(map_synchro_status)
......@@ -28,12 +27,12 @@ def get_synchro_state_file(id):
return synchro_state_name
def read_synchro_status(id):
def read_synchro_status(queue, id):
map_synchro_status = {}
synchro_state_file = get_synchro_state_file(id)
try:
content = read_synchro_state_file(synchro_state_file)
content = read_synchro_state_file(queue, synchro_state_file)
lines = content.split("\n")
for line in lines:
......@@ -45,17 +44,17 @@ def read_synchro_status(id):
return map_synchro_status
def move_synchro_listing_to_download(download_id):
listing_file = read_synchro_status(download_id)["Output"]
def move_synchro_listing_to_download(queue, download_id):
listing_file = read_synchro_status(queue, download_id)["Output"]
is_success = copy_listing_file(
is_success = copy_listing_file(queue,
listing_file, get_workspace_dir_name(download_id))
return is_success
class Synchro(Process):
def __init__(self, queue, xml_filename, download_id, modeDownload=False):
def __init__(self, celery_queue, queue, xml_filename, download_id, modeDownload=False):
super(Synchro, self).__init__()
current_time = datetime.datetime.utcnow()
#self.path= settings.CACHE_SYNCHRO_PATH + download_id + "/" + current_time.strftime('%Y/%m/%d')
......@@ -65,6 +64,7 @@ class Synchro(Process):
self.modeDownload = modeDownload
self.queue = queue
self.celery_queue = celery_queue
self.configuration_filename = xml_filename
self.listing_filename = synchro_output_filename
......@@ -72,12 +72,14 @@ class Synchro(Process):
self.synchro_subpath = get_workspace_dir_name(download_id) \
+ SYNCHRO_HISTORY + current_time.strftime('%Y%m%d')
def check_status_synchro(self):
return check_status_synchro(self.job_filename)
return check_status_synchro(self.celery_queue, self.job_filename)
def run(self):
is_success = run_synchro(self.configuration_filename,
is_success = run_synchro(self.celery_queue,
self.configuration_filename,
self.listing_filename,
self.job_filename,
self.synchro_subpath)
......
from django.conf import settings
import os
import downloader_django.common.constants
from downloader_django.common.downloaderUtil import get_workspace_dir_name
from downloader_django.common.system.folder import Folder
from downloader_django.common.system.file import File
from downloader_django.common.system.tasks import write_content_file, remove_file, rename_a_file, copy_file
from downloader_django.common.system.tasks import read_test_state_file, read_manual_listing, write_manual_listing
def add_test_status(file_map):
def add_test_status(queue, file_map):
list_map_filename = file_map["TEST"]
for map_filename in list_map_filename:
for filename, values in list(map_filename.items()):
mapFileName = values[0]
map_test_status = {}
id = mapFileName.attrib['id']
#test_state_name = "%s-%s.test"%(id, gethostname())
test_state_name = "%s.test" % (id)
test_state_path = os.path.join(
settings.CACHE_JOB_PATH, test_state_name)
f = File(test_state_path)
try:
f.open(mode='r')
lines = f.readlines()
f.close()
for line in lines:
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_test_status[lineSplit[0].strip()] = lineSplit[1]
except OSError:
pass
values.append(map_test_status)
download_id = mapFileName.attrib['id']
map_test_status = read_test_status(queue, download_id)
def get_test_state_file(fileid):
test_state_name = "%s.test" % (fileid)
test_state_path = os.path.join(settings.CACHE_JOB_PATH, test_state_name)
return test_state_path
values.append(map_test_status)
def read_test_status(fileid):
def read_test_status(queue, download_id):
map_test_status = {}
test_state_file = get_test_state_file(fileid)
f = File(test_state_file)
try:
f.open(mode='r')
lines = f.readlines()
f.close()
content = read_test_state_file(queue, download_id + ".test")
lines = content.split("\n")
for line in lines:
lineSplit = line.split("=")
if len(lineSplit) > 1:
......@@ -57,34 +32,20 @@ def read_test_status(fileid):
class TestFile:
def __init__(self, download_id):
folder = Folder(os.path.join(settings.WORKSPACE_DOWNLOAD, get_workspace_dir_name(
download_id) + downloader_django.common.constants.LISTING_MANUAL), create=True)
self._filetest = os.path.join(
str(folder.getPath()), "listing.list.test")
def __init__(self, queue, download_id):
self._id = download_id
self._filename = "listing.list.test"
self._queue = queue
def get_content_test_file(self):
f = File(self._filetest)
if f.exist():
f.open(mode='r')
else:
write_content_file(self._filetest, "")
f.open(mode='r')
filenames = f.readlines()
f.close()
content = read_manual_listing(self._queue, self._filename, self._id)
filenames = content.split("\n")
return filenames
def save_content_test_file(self, test_filename, xml_filename):
is_success = False
f = File(self._filetest)
is_success = write_content_file(self._filetest, test_filename)
f.removeLockFileExist(settings.LOCK_FILE_EXTENSION)
if is_success:
downloadConfig = os.path.join(settings.CONF_PATH, xml_filename)
tmp_file = downloadConfig + ".tmp"
is_success = rename_a_file(downloadConfig, tmp_file)
if is_success:
is_success = copy_file(tmp_file, downloadConfig)
remove_file(tmp_file)