Unverified Commit cd2cf20f authored by CHARLES's avatar CHARLES 🐧
Browse files

working hard or hardly working

parent 30d0e7ed
include *.txt
recursive-include downloader_django/locale *
recursive-include downloader_django/static *
recursive-include downloader_django/templates *
recursive-include lib/downloader_django/locale *
recursive-include lib/downloader_django/static *
recursive-include lib/downloader_django/templates *
include settings.py
recursive-include downloader *.py
......@@ -36,7 +36,6 @@ URL_PREFIX = os.environ.get('URL_PREFIX', '')
PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
DEBUG = True
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', 'your_email@example.com'),
......@@ -46,7 +45,7 @@ MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'db/downloader.db', # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
'USER': '',
......@@ -118,31 +117,21 @@ STATICFILES_DIRS = (
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
LOCALE_PATHS = (
os.path.join(STATIC_ROOT, 'locale')
os.path.join(STATIC_ROOT, 'locale'),
)
# Make this unique, and don't share it with anybody.
SECRET_KEY = 't25jsr7eh9!it09qhw4!^%zg)0&)l3#g^!pyj!#0w+^@2*3x9h'
# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
# 'django.template.loaders.eggs.Loader',
)
TEMPLATE_CONTEXT_PROCESSORS = (
"django.contrib.auth.context_processors.auth",
"django.core.context_processors.debug",
"django.core.context_processors.i18n",
"django.core.context_processors.media",
"django.core.context_processors.static",
"django.core.context_processors.tz",
"django.contrib.messages.context_processors.messages"
)
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'APP_DIRS': True,
},
]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
......@@ -155,23 +144,16 @@ MIDDLEWARE_CLASSES = (
)
# Lien vers la page si l'authentification echoue
LOGIN_URL='%saccount/login/' % URL_PREFIX
LOGIN_URL = '%saccount/login/' % URL_PREFIX
# Lien vers la page si l'authentification reussi
LOGIN_REDIRECT_URL='%sdownloaderadmin/' % URL_PREFIX
LOGIN_REDIRECT_URL = '%sdownloaderadmin/' % URL_PREFIX
ROOT_URLCONF = 'downloader_django.urls'
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'downloader_django.wsgi.application'
TEMPLATE_DIRS = (
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
pkg_file('downloader_django', 'templates'),
)
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
......@@ -218,32 +200,31 @@ LOGGING = {
}
#infile_path = os.path.join(APPDATA, 'controller/sc_path.xml')
#with open(infile_path, 'r') as infile:
# with open(infile_path, 'r') as infile:
# paths = objectify.parse(infile).getroot()
#CONF_PATH= os.path.join(WORKSPACE if paths.downloads_config.get('location') == "workspace" else APPDATA,
# CONF_PATH= os.path.join(WORKSPACE if paths.downloads_config.get('location') == "workspace" else APPDATA,
# str(paths.downloads_config))
#WORKSPACE_DOWNLOAD= os.path.join(WORKSPACE if paths.work.get('location') == "workspace" else APPDATA,
# WORKSPACE_DOWNLOAD= os.path.join(WORKSPACE if paths.work.get('location') == "workspace" else APPDATA,
# str(paths.work))
#LOG_PATH = os.path.join(WORKSPACE if paths.process_log_archive.get('location') == "workspace" else APPDATA,
# LOG_PATH = os.path.join(WORKSPACE if paths.process_log_archive.get('location') == "workspace" else APPDATA,
# str(paths.process_log_archive))
#CACHE_JOB_PATH = os.path.join(WORKSPACE if paths.jobfile.get('location') == "workspace" else APPDATA,
# CACHE_JOB_PATH = os.path.join(WORKSPACE if paths.jobfile.get('location') == "workspace" else APPDATA,
# str(paths.jobfile))
#CACHE_PROVIDER_PATH = os.path.join(WORKSPACE if paths.providerfile.get('location') == "workspace" else APPDATA,
# CACHE_PROVIDER_PATH = os.path.join(WORKSPACE if paths.providerfile.get('location') == "workspace" else APPDATA,
# str(paths.providerfile))
#BIN_PATH= str(paths.exec_path)
#CACHE_SYNCHRO_PATH = os.path.join(WORKSPACE, 'cache/synchrofile/')
#infile_path = os.path.join(APPDATA, 'controller/sc_global.xml')
#with open(infile_path, 'r') as infile:
# with open(infile_path, 'r') as infile:
# global_config = objectify.parse(infile).getroot()
#LOCK_FILE_EXTENSION = global_config.lockfile_extension
# This loads the celery configuration from FELYX_CELERY_CFG file
with open(os.environ['CELERY_CFG'], 'r') as f:
with open(os.environ['CELERY_CFG'], 'rb') as f:
CELERY_CONF = simplejson.loads(f.read())
CELERY = Celery()
CELERY.conf.update(**CELERY_CONF)
../downloader_django/static/
\ No newline at end of file
../lib/downloader_django/static
\ No newline at end of file
from django import forms
import os
class InputFileForm(forms.Form):
data_filename = forms.FileField(label="filepath * ") # for creating file input
from django.conf.urls import patterns, include, url
urlpatterns = patterns('downloader_django.apps.downloader_admin.views',
url(r'^$'
, 'listdownload'
, name='listdownload'),
url(r'^test$'
, 'test'
, name='test'),
url(r'^new$'
, 'new'
, name='new'),
url(r'^nbActivatedFile$'
, 'getActivatedNbFile'
, name='nbActivatedFile'),
url(r'^nbDesactivatedFile$'
, 'getDesactivatedNbFile'
, name='nbDeactivatedFile'),
url(r'^nbTestFile$'
, 'getTestNbFile'
, name='nbTestFile'),
url(r'^help$'
, 'help'
, name='help'),
url(r'^filterdownloads/(?P<pattern>\S+)$'
, 'filter_downloads'
, name='filterdownloads'),
url(r'^createdownload/(?P<download_id>\S+)$'
, 'create_and_edit_download'
, name='createdownload'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/define_test_file$'
, 'define_test_file'
, name='define_test_file'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/save_test_file$'
, 'save_test_file'
, name='save_test_file'),
url(r'^(?P<download_id>\S+)/testlog$'
, 'display_test_log'
, name='testlog'),
url(r'^(?P<download_id>\S+)/synchrolog$'
, 'display_synchro_log'
, name='synchrolog'),
url(r'^(?P<download_id>\S+)/logfile$'
, 'display_download_log'
, name='display_download_log'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/copy_as_file'
, 'copy_as_file'
, name='copy_as_file'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/reload_from_file'
, 'reload_from_file'
, name='reload_from_file'),
url(r'^synchro/listing/download/(?P<download_id>\S+)'
, 'move_synchro_listing_to_download'
, name='move_synchro_listing_to_download'),
url(r'^synchro/listing/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'display_synchro_listing'
, name='display_synchro_listing'),
url(r'^synchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'status_synchro'
, name='status_synchro'),
url(r'^test/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'status_test'
, name='status_test'),
url(r'^nbfile/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'nb_file_synchro'
, name='nb_file_synchro'),
url(r'^date/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'date_synchro'
, name='date_synchro'),
url(r'^checksynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'check_synchro'
, name='checksynchro'),
url(r'^runsynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'run_synchro'
, name='runsynchro'),
url(r'^(?P<xml_filename>\S+)/save$'
, 'save'
, name='save'),
url(r'^(?P<xml_filename>\S+)/activate$'
, 'activate'
, name='activate'),
url(r'^(?P<xml_filename>\S+)/desactivate$'
, 'desactivate'
, name='deactivate'),
url(r'^(?P<xml_filename>\S+)/delete$'
, 'delete'
, name='delete'),
url(r'^(?P<xml_filename>\S+)/modetest$'
, 'modetest'
, name='modetest'),
url(r'^(?P<xml_filename>\S+)$'
, 'editdownload'
, name='edit'),
)
from django.conf.urls import patterns, include, url
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.core.urlresolvers import reverse
# Uncomment the next two lines to enable the admin:
from django.contrib import admin, auth
admin.autodiscover()
import os
url_prefix = os.environ.get('URL_PREFIX', '')
urlpatterns = patterns('',
# Examples:
# url(r'^$', 'django_downloader.views.home', name='home'),
#url(r'^django_downloader/', include('django_downloader.urls')),
url( r'%sadmin' % url_prefix
, include(admin.site.urls)
),
url( r'^%saccount/login' % url_prefix
, 'django.contrib.auth.views.login'
, name='login'
),
url( r'^%saccount/logout' % url_prefix
, 'django.contrib.auth.views.logout'
, {'next_page': '%saccount/login' % url_prefix}
, name='logout'
),
url( r'%sdownloaderadmin/' % url_prefix
, include( 'downloader_django.apps.downloader_admin.urls'
, namespace='dladmin'
)
),
#
# url( r'%smonitoring/' % url_prefix
# , include( 'downloader_django.apps.monitor.urls'
# , namespace='monitor'
# )
# ),
# Uncomment the admin/doc line below to enable admin documentation:
# url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
# Uncomment the next line to enable the admin:
# url(r'^admin/', include(admin.site.urls)),
)
urlpatterns += staticfiles_urlpatterns()
name: downloader_ui
channels:
- defaults
dependencies:
- python>=3.7
- gcc
from django import forms
class InputFileForm(forms.Form):
data_filename = forms.FileField(
label="filepath * ") # for creating file input
from django.conf.urls import url
from . import views
urlpatterns = [
url(r'^$', views.listdownload, name='listdownload'),
url(r'^test$', views.test, name='test'),
url(r'^new$', views.new, name='new'),
url(r'^nbActivatedFile$', views.getActivatedNbFile, name='nbActivatedFile'),
url(r'^nbDesactivatedFile$', views.getDesactivatedNbFile, name='nbDeactivatedFile'),
url(r'^nbTestFile$', views.getTestNbFile, name='nbTestFile'),
url(r'^help$', views.help, name='help'),
url(r'^filterdownloads/(?P<pattern>\S+)$', views.filter_downloads, name='filterdownloads'),
url(r'^createdownload/(?P<download_id>\S+)$', views.create_and_edit_download, name='createdownload'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/define_test_file$', views.define_test_file, name='define_test_file'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/save_test_file$', views.save_test_file, name='save_test_file'),
url(r'^(?P<download_id>\S+)/testlog$', views.display_test_log, name='testlog'),
url(r'^(?P<download_id>\S+)/synchrolog$', views.display_synchro_log, name='synchrolog'),
url(r'^(?P<download_id>\S+)/logfile$', views.display_download_log, name='display_download_log'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/copy_as_file', views.copy_as_file, name='copy_as_file'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/reload_from_file', views.reload_from_file, name='reload_from_file'),
url(r'^synchro/listing/download/(?P<download_id>\S+)', views.move_synchro_listing_to_download, name='move_synchro_listing_to_download'),
url(r'^synchro/listing/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.display_synchro_listing, name='display_synchro_listing'),
url(r'^synchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.status_synchro, name='status_synchro'),
url(r'^test/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.status_test, name='status_test'),
url(r'^nbfile/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.nb_file_synchro, name='nb_file_synchro'),
url(r'^date/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.date_synchro, name='date_synchro'),
url(r'^checksynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.check_synchro, name='checksynchro'),
url(r'^runsynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$', views.run_synchro, name='runsynchro'),
url(r'^(?P<xml_filename>\S+)/save$', views.save, name='save'),
url(r'^(?P<xml_filename>\S+)/activate$', views.activate, name='activate'),
url(r'^(?P<xml_filename>\S+)/desactivate$', views.desactivate, name='deactivate'),
url(r'^(?P<xml_filename>\S+)/delete$', views.delete, name='delete'),
url(r'^(?P<xml_filename>\S+)/modetest$', views.modetest, name='modetest'),
url(r'^(?P<xml_filename>\S+)$', views.editdownload, name='edit'),
]
......@@ -5,7 +5,6 @@ from multiprocessing import Process, Queue
from downloader_django.common.constants import SYNCHRO_HISTORY
from downloader_django.common.downloaderUtil import get_workspace_dir_name
from downloader_django.common.system.tasks import run_synchro, read_synchro_state_file, \
copy_listing_file, check_status_synchro
......@@ -14,64 +13,67 @@ from downloader_django.common.system.tasks import run_synchro, read_synchro_stat
def add_synchro_status(file_map):
list_map_filename = file_map["ON"]
for map_filename in list_map_filename:
for filename, values in map_filename.items():
if len(values)>0:
mapFileName = values[0]
for filename, values in list(map_filename.items()):
if len(values) > 0:
mapFileName = values[0]
id = mapFileName.attrib['id']
map_synchro_status = read_synchro_status(id)
values.append(map_synchro_status)
def get_synchro_state_file(id):
synchro_state_name = "%s.synchro.job"%(id)
synchro_state_name = "%s.synchro.job" % (id)
return synchro_state_name
def read_synchro_status(id):
map_synchro_status={}
map_synchro_status = {}
synchro_state_file = get_synchro_state_file(id)
try:
content = read_synchro_state_file(synchro_state_file)
lines = content.split("\n")
for line in lines :
for line in lines:
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_synchro_status[lineSplit[0].strip()] = lineSplit[1].strip()
map_synchro_status[lineSplit[0].strip()] = lineSplit[1].strip()
except OSError:
pass
pass
return map_synchro_status
def move_synchro_listing_to_download(download_id):
listing_file = read_synchro_status(download_id)["Output"]
is_success = copy_listing_file(listing_file, get_workspace_dir_name(download_id))
is_success = copy_listing_file(
listing_file, get_workspace_dir_name(download_id))
return is_success
class Synchro(Process):
def __init__(self,queue, xml_filename, download_id, modeDownload=False) :
def __init__(self, queue, xml_filename, download_id, modeDownload=False):
super(Synchro, self).__init__()
current_time=datetime.datetime.utcnow()
current_time = datetime.datetime.utcnow()
#self.path= settings.CACHE_SYNCHRO_PATH + download_id + "/" + current_time.strftime('%Y/%m/%d')
name = download_id + "_" + current_time.strftime('%Y%m%d_%H%M%S')
synchro_output_filename= "%s.synchro" % (name)
self.modeDownload=modeDownload
synchro_output_filename = "%s.synchro" % (name)
self.modeDownload = modeDownload
self.queue = queue
self.configuration_filename = xml_filename
self.listing_filename = synchro_output_filename
self.job_filename = download_id + ".synchro.job"
self.job_filename = download_id + ".synchro.job"
self.synchro_subpath = get_workspace_dir_name(download_id) \
+ SYNCHRO_HISTORY + current_time.strftime('%Y%m%d')
+ SYNCHRO_HISTORY + current_time.strftime('%Y%m%d')
def check_status_synchro(self):
return check_status_synchro(self.job_filename)
def run(self):
......@@ -80,5 +82,4 @@ class Synchro(Process):
self.job_filename,
self.synchro_subpath)
self.queue.put(is_success)
self.queue.put(is_success)
......@@ -13,77 +13,81 @@ from downloader_django.common.system.tasks import write_content_file, remove_fil
def add_test_status(file_map):
list_map_filename = file_map["TEST"]
for map_filename in list_map_filename:
for filename, values in map_filename.items():
mapFileName = values[0]
map_test_status={}
for filename, values in list(map_filename.items()):
mapFileName = values[0]
map_test_status = {}
id = mapFileName.attrib['id']
#test_state_name = "%s-%s.test"%(id, gethostname())
test_state_name = "%s.test"%(id)
test_state_path = os.path.join(settings.CACHE_JOB_PATH, test_state_name)
f= File(test_state_path)
test_state_name = "%s.test" % (id)
test_state_path = os.path.join(
settings.CACHE_JOB_PATH, test_state_name)
f = File(test_state_path)
try:
f.open(mode='r')
lines = f.readlines()
f.close()
for line in lines :
for line in lines:
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_test_status[lineSplit[0].strip()] = lineSplit[1]
except OSError:
pass
pass
values.append(map_test_status)
def get_test_state_file(fileid):
test_state_name = "%s.test"%(fileid)
test_state_name = "%s.test" % (fileid)
test_state_path = os.path.join(settings.CACHE_JOB_PATH, test_state_name)
return test_state_path
def read_test_status(fileid):
map_test_status={}
map_test_status = {}
test_state_file = get_test_state_file(fileid)
f= File(test_state_file)
f = File(test_state_file)
try:
f.open(mode='r')
lines = f.readlines()
lines = f.readlines()
f.close()
for line in lines :
for line in lines:
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_test_status[lineSplit[0].strip()] = lineSplit[1].strip()
map_test_status[lineSplit[0].strip()] = lineSplit[1].strip()
except OSError:
pass
pass
return map_test_status
class TestFile :
def __init__(self,download_id) :
folder = Folder(os.path.join(settings.WORKSPACE_DOWNLOAD, get_workspace_dir_name(download_id) + downloader_django.common.constants.LISTING_MANUAL), create=True)
self._filetest = os.path.join(str(folder.getPath()), "listing.list.test")
def get_content_test_file(self,):
filename=""
f= File(self._filetest)
class TestFile:
def __init__(self, download_id):
folder = Folder(os.path.join(settings.WORKSPACE_DOWNLOAD, get_workspace_dir_name(
download_id) + downloader_django.common.constants.LISTING_MANUAL), create=True)
self._filetest = os.path.join(
str(folder.getPath()), "listing.list.test")
def get_content_test_file(self):
f = File(self._filetest)
if f.exist():
f.open(mode='r')
else:
is_success = write_content_file(self._filetest, "")
write_content_file(self._filetest, "")
f.open(mode='r')
filenames = f.readlines()
f.close()
return filenames
def save_content_test_file(self, test_filename, xml_filename):
is_success=False
f= File(self._filetest)
is_success = write_content_file(self._filetest, test_filename)
is_success = False
f = File(self._filetest)
is_success = write_content_file(self._filetest, test_filename)
f.removeLockFileExist(settings.LOCK_FILE_EXTENSION)
if is_success:
downloadConfig = os.path.join(settings.CONF_PATH, xml_filename)
tmp_file= downloadConfig+ ".tmp"
is_success = rename_a_file(downloadConfig,tmp_file)
tmp_file = downloadConfig + ".tmp"
is_success = rename_a_file(downloadConfig, tmp_file)
if is_success:
is_success = copy_file(tmp_file, downloadConfig)
remove_file(tmp_file)
return is_success
......@@ -12,7 +12,6 @@ import logging
from downloader_django.common.system.tasks import write_content_file
class XMLWriter(object):
"""
Class XMLWriter
......@@ -25,12 +24,12 @@ class XMLWriter(object):
pass
def write(self, xmlTree, file):
is_success=False
is_success = False