Commit 13987315 authored by pm22d12's avatar pm22d12
Browse files

Fix #3 & add ftps support

parent 3d03606c
{ "broker_url": "amqp://zenika_training:zenika_training@vrabbitmq1-val.ifremer.fr:5672/downloader",
{ "management_url" : "https://rabbitmq-management-val.ifremer.fr:15672",
"broker_url": "amqp://cersat-downloader-ui:Bruce and the Spatules@rabbitmq-cluster-val.ifremer.fr:5672/cersat-downloader",
"broker_heartbeat": 30,
"imports": [],
"result_backend": "amqp",
......
......@@ -4,7 +4,7 @@ from . import views
urlpatterns = [
url(r'^$', views.listdownload, name='listdownload'),
url(r'^test$', views.test, name='test'),
# url(r'^test$', views.test, name='test'),
url(r'^new$', views.new, name='new'),
url(r'^nbActivatedFile$', views.getActivatedNbFile, name='nbActivatedFile'),
url(r'^nbDesactivatedFile$', views.getDesactivatedNbFile, name='nbDeactivatedFile'),
......@@ -39,6 +39,7 @@ urlpatterns = [
url(r'^(?P<xml_filename>\S+)/activate$', views.activate, name='activate'),
url(r'^(?P<xml_filename>\S+)/desactivate$', views.desactivate, name='deactivate'),
url(r'^remove/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.remove, name='remove'),
url(r'^(?P<xml_filename>\S+)/delete$', views.delete, name='delete'),
url(r'^(?P<xml_filename>\S+)/modetest$', views.modetest, name='modetest'),
......
......@@ -147,6 +147,35 @@ def get_downloads(queue):
return mapFile, map_file_id, map_file_id_test
def get_downloads_id(queue):
map_id = {}
for filename in tasks.get_configuration_files(queue):
name, fileExtension = os.path.splitext(filename)
filesplit = name.split(".")
file_data = get_xml_data(queue, filename)
id = file_data.attrib['id']
state = None
if fileExtension == ".xml" and filesplit[-1] == "download":
state = "ON"
elif fileExtension == ".xml" and filesplit[-1] == "test" and filesplit[-2] == "download":
state = "TEST"
elif fileExtension in (".OFF", ".BAD") and filesplit[-1] == "xml" and filesplit[-2] == "download":
state = "OFF"
if state is not None:
try:
file_data = get_xml_data(queue, filename)
id = file_data.attrib['id']
except XMLSyntaxError:
pass
map_id[id] = state
return map_id
def filter_downloads(queue, pattern):
map_downloads = {}
......@@ -467,9 +496,13 @@ def save_data_source(file_data, submitted_data):
dummy = etree.SubElement(parent_sel.opensearch, "area")
parent_sel.opensearch.area = submitted_data['opensearch_area']
if parent_sel.opensearch.find("request_format") is None:
dummy = etree.SubElement(parent_sel.opensearch, "request_format")
parent_sel.opensearch.request_format = submitted_data['opensearch_format']
if 'opensearch_format' in submitted_data:
if parent_sel.opensearch.find("request_format") is None:
dummy = etree.SubElement(parent_sel.opensearch, "request_format")
parent_sel.opensearch.request_format = submitted_data['opensearch_format']
else:
if parent_sel.opensearch.find("request_format") is not None:
parent_sel.opensearch.remove(parent_sel.opensearch.request_format)
def save_data_destination(file_data, submitted_data):
......@@ -647,3 +680,11 @@ def save_download_settings(file_data, submitted_data):
parent.monitoring = "true"
else:
parent.monitoring = "false"
if submitted_data['project_name'] != "":
if parent.find("project_name") is None:
dummy = etree.SubElement(parent, "project_name")
parent.project_name = submitted_data['project_name']
elif parent.find("project_name") is not None:
parent.remove(parent.project_name)
......@@ -7,6 +7,7 @@ DATE_FORMAT_FRONT = '%Y-%m-%d'
EXTENSION_ACTIVE = ".download.xml"
EXTENSION_DISABLED = ".download.xml.OFF"
EXTENSION_BAD = ".download.xml.BAD"
EXTENSION_JOB = ".job"
EXTENSION_LISTING = ".list"
EXTENSION_TEST = ".download.test.xml"
......
......@@ -185,7 +185,7 @@ msgstr "Change the user group of a file.\n"
#: ====== tab Advanced
#: ==========
#: ========== Misc parameters
#: ========== Control parameters
#: ================
#: apps/downloader_admin/views.py:366
#: templates/adminview/editdownload.html:555
......@@ -210,9 +210,6 @@ msgstr "Number of tries to download a file when error occurs"
msgid "check_provider_before_download.help"
msgstr "Assert that the provider is available before starting crawl loop"
msgid "monitoring_enabled.help"
msgstr "Enable / disable sending monitoring information"
#: ========== Scanning / listing parameters
#: ================
......@@ -228,10 +225,15 @@ msgstr "Preserve or not the most recent scan"
msgid "to_download_max_nbr.help"
msgstr "Used to validate the listing of data\nto download generated after a remote crawl"
msgid "to_download_max_nbr_cg.help"
msgstr "*** Used to validate the listing of data to download generated after a remote crawl ***"
#: ========== Miscellaneous parameters
#: ================
msgid "monitoring_enabled.help"
msgstr "Enable / disable sending monitoring information"
msgid "project_name.help"
msgstr "Name of associated project"
msgid "max_activated_flow.help"
msgstr "It defines the maximum of connection on server.\n"
"(excepted 2 connections for check provider) "
"(In addition to the connection reserved for check the provider) "
......@@ -93,14 +93,15 @@
<xs:simpleType name="remoteTypeEnum">
<xs:restriction base="xs:string">
<xs:enumeration value='Localpath'/>
<xs:enumeration value='Ftp'/>
<xs:enumeration value='Localpointer'/>
<xs:enumeration value='Localmove'/>
<xs:enumeration value='Ftps'/>
<xs:enumeration value='Sftp'/>
<xs:enumeration value='Https'/>
<xs:enumeration value='Https_opensearch'/>
<xs:enumeration value='Webdav'/>
<xs:enumeration value='Localpath'/>
<xs:enumeration value='Localpointer'/>
<xs:enumeration value='Localmove'/>
<xs:enumeration value='Onlynotify'/>
</xs:restriction>
</xs:simpleType>
......@@ -250,6 +251,7 @@
<xs:element name="max_nb_of_concurrent_streams" type="positiveIntegerOrNull" minOccurs="0" maxOccurs="1"/>
<xs:element name="check_source_availability" type="booleanOrNull" minOccurs="0" maxOccurs="1"/>
<xs:element name="monitoring" type="booleanOrNull" minOccurs="0" maxOccurs="1"/>
<xs:element name="project_name" type="xs:string" minOccurs="0" maxOccurs="1"/>
</xs:all>
</xs:complexType>
......
......@@ -38,7 +38,7 @@
<div class="controls">
<input class="input-xlarge" type="text" id="inputError" name="newfile"
placeholder="download name">
<span class="help-inline">Unknown error</span>
<span class="help-inline">{{ file_error }}</span> <!--Unknown error-->
</div>
</div>
{% endif %}
......
......@@ -1311,7 +1311,8 @@
function getRequestDeleteDownloadRow(filename, fileid, source) {
fileid = getContextId(_NONE_, fileid);
var requesturl = BASE_URL + filename + "/delete";
/*var requesturl = BASE_URL + filename + "/delete";*/
var requesturl = BASE_URL + "remove" + "/" +filename + "/" + fileid;
var test = $.get(requesturl, function(data) {
if (data == _NO_ERROR) {
......
......@@ -182,11 +182,13 @@ def get_active_queues(context):
else:
user = netloc[:dpoint]
psswd = netloc[dpoint+1:at]
host = netloc[at+1:]
dpoint = host.find(':')
if dpoint != -1:
host = host[:dpoint]
url_rmq = f"http://{host}:15672/api/queues/{spliturl.path[1:]}"
#host = netloc[at+1:]
#dpoint = host.find(':')
#if dpoint != -1:
# host = host[:dpoint]
#url_rmq = f"http://{host}:15672/api/queues/{spliturl.path[1:]}"
host = settings.CELERY.conf['management_url']
url_rmq = f"{host}/api/queues/{spliturl.path[1:]}"
response = requests.get(url_rmq, auth=(user, psswd)).json()
celery_queues = [q['name'] for q in response]
downloader_queues = [(q['arguments']['alias'], q['name']) for q in response
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment