Commit 59891d5f authored by MAISSIAT's avatar MAISSIAT
Browse files

Fist commit

parent eccfcfaf
from django import forms
import os
from django.db import models
# Create your models here.
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
from django.conf.urls import patterns, include, url
urlpatterns = patterns('downloader_django.apps.downloader_admin.views',
url(r'^$'
, 'listdownload'
, name='listdownload'),
url(r'^test$'
, 'test'
, name='test'),
url(r'^new$'
, 'new'
, name='new'),
url(r'^nbActivatedFile$'
, 'getActivatedNbFile'
, name='nbActivatedFile'),
url(r'^nbDesactivatedFile$'
, 'getDesactivatedNbFile'
, name='nbDeactivatedFile'),
url(r'^nbTestFile$'
, 'getTestNbFile'
, name='nbTestFile'),
url(r'^help$'
, 'help'
, name='help'),
url(r'^filterdownloads/(?P<pattern>\S+)$'
, 'filter_downloads'
, name='filterdownloads'),
url(r'^createdownload/(?P<download_id>\S+)$'
, 'create_and_edit_download'
, name='createdownload'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/define_test_file$'
, 'define_test_file'
, name='define_test_file'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/save_test_file$'
, 'save_test_file'
, name='save_test_file'),
url(r'^(?P<download_id>\S+)/testlog$'
, 'display_test_log'
, name='testlog'),
url(r'^(?P<download_id>\S+)/synchrolog$'
, 'display_synchro_log'
, name='synchrolog'),
url(r'^(?P<download_id>\S+)/logfile$'
, 'display_download_log'
, name='display_download_log'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/copy_as_file'
, 'copy_as_file'
, name='copy_as_file'),
url(r'^(?P<xml_filename>\S+)/(?P<download_id>\S+)/reload_from_file'
, 'reload_from_file'
, name='reload_from_file'),
url(r'^synchro/listing/download/(?P<download_id>\S+)'
, 'move_synchro_listing_to_download'
, name='move_synchro_listing_to_download'),
url(r'^synchro/listing/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'display_synchro_listing'
, name='display_synchro_listing'),
url(r'^synchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'status_synchro'
, name='status_synchro'),
url(r'^test/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'status_test'
, name='status_test'),
url(r'^nbfile/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'nb_file_synchro'
, name='nb_file_synchro'),
url(r'^date/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'date_synchro'
, name='date_synchro'),
url(r'^checksynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'check_synchro'
, name='checksynchro'),
url(r'^runsynchro/(?P<xml_filename>\S+)/(?P<download_id>\S+)$'
, 'run_synchro'
, name='runsynchro'),
url(r'^(?P<xml_filename>\S+)/save$'
, 'save'
, name='save'),
url(r'^(?P<xml_filename>\S+)/activate$'
, 'activate'
, name='activate'),
url(r'^(?P<xml_filename>\S+)/desactivate$'
, 'desactivate'
, name='deactivate'),
url(r'^(?P<xml_filename>\S+)/delete$'
, 'delete'
, name='delete'),
url(r'^(?P<xml_filename>\S+)/modetest$'
, 'modetest'
, name='modetest'),
url(r'^(?P<xml_filename>\S+)$'
, 'editdownload'
, name='edit'),
)
This diff is collapsed.
from celery.execute import send_task
from django.conf import settings
from multiprocessing import Process, Queue
from socket import gethostname
from subprocess import Popen, PIPE
import datetime
import os
import time
import downloader_django.common.constants
from downloader_django.common.downloaderUtil import get_workspace_dir_name
from downloader_django.common.system.file import File
from downloader_django.common.system.tasks import copy_file, write_content_file, run_synchro, read_synchro_state_file
def add_synchro_status(file_map):
list_map_filename = file_map["ON"]
for map_filename in list_map_filename:
for filename, values in map_filename.items():
if len(values)>0:
mapFileName = values[0]
id = mapFileName.attrib['id']
#synchro_state_name = "%s-%s.synchro.job"%(id, gethostname())
map_synchro_status = read_synchro_status(id)
values.append(map_synchro_status)
def get_synchro_state_file(id):
synchro_state_name = "%s.synchro.job"%(id)
#synchro_state_file = os.path.join(settings.CACHE_SYNCHRO_PATH, synchro_state_name)
#return synchro_state_file
return synchro_state_name
def read_synchro_status(id):
map_synchro_status={}
synchro_state_file = get_synchro_state_file(id)
#f= File(synchro_state_file)
try:
#f.open(mode='r')
content = read_synchro_state_file(synchro_state_file)
lines = content.split("\n")
#lines = f.readlines()
#f.close()
for line in lines :
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_synchro_status[lineSplit[0].strip()] = lineSplit[1].strip()
except OSError:
pass
return map_synchro_status
def move_synchro_listing_to_download(download_id):
map_synchro_status= read_synchro_status(download_id)
listing_file = read_synchro_status(download_id)["Output"]
listing_file_path, listing_file_name = os.path.split(listing_file)
copy_path = os.path.join(settings.WORKSPACE_DOWNLOAD, os.path.join("downloads",
get_workspace_dir_name(download_id) + downloader_django.common.constants.LISTINGS_AUTO))
copy_file_name=os.path.join(copy_path, listing_file_name + downloader_django.common.constants.EXTENSION_LISTING)
#is_success = copy_file(listing_file, copy_file_name)
is_success = copy_listing_file(listing_file, get_workspace_dir_name(download_id))
return is_success
class Synchro(Process):
def __init__(self,queue, xml_filename, download_id, modeDownload=False) :
super(Synchro, self).__init__()
#Process.__init__(self)
current_time=datetime.datetime.utcnow()
self.path= settings.CACHE_SYNCHRO_PATH + download_id + "/" + current_time.strftime('%Y/%m/%d')
name = download_id + "_" + current_time.strftime('%Y%m%d_%H%M%S')
#synchro_output_filename= "%s-%s.synchro" % (name, gethostname())
synchro_output_filename= "%s.synchro" % (name)
self.output_file = os.path.join(str(self.path), synchro_output_filename)
self.conf_file = os.path.join(settings.CONF_PATH, xml_filename)
#self.synchro_job_file=downloader_django.common.constants.CACHE_SYNCHRO_PATH + name + ".synchro.job"
self.synchro_job_file=settings.CACHE_SYNCHRO_PATH + download_id + ".synchro.job"
self.modeDownload=modeDownload
self.queue = queue
self.configuration_filename = xml_filename
self.listing_filename = synchro_output_filename
self.job_filename = download_id + ".synchro.job"
self.synchro_subpath = download_id + "/" + current_time.strftime('%Y/%m/%d')
def get_synchro_job_file(self):
return self.synchro_job_file
def run(self):
#argsList = ["-c", self.conf_file]
argsList = ["-c"]
argsList.append(self.configuration_filename)
argsList.append("-o")
#argsList.append(self.output_file)
argsList.append(self.listing_filename)
argsList.append("-j")
#argsList.append(self.synchro_job_file)
argsList.append(self.job_filename)
argsList.append("--history-path")
#argsList.append(self.path)
argsList.append(self.synchro_subpath)
if self.modeDownload:
argsList.append("--with-download")
cmd = [settings.BIN_PATH + '/downloadsync']+argsList
#result = settings.CELERY.send_task('felyx_work.processors.celery.wrapper.run', ((" ").join(cmd), ''))
is_success = run_synchro(self.configuration_filename,
self.listing_filename,
self.job_filename,
self.synchro_subpath)
#result = settings.CELERY.send_task('downloader_celery_tasks.run_synchro', argsList)
#out = result.get()
#is_success = result.info
#is_success = not out
self.queue.put(is_success)
from celery.exceptions import TimeoutError
from django.conf import settings
from lxml import etree, objectify
from lxml.etree import XMLSyntaxError
import os
import downloader_django.common.constants
from downloader_django.common.downloaderUtil import get_workspace_dir_name
from downloader_django.common.system.folder import Folder
from downloader_django.common.system.file import File
from downloader_django.common.system.tasks import write_content_file, remove_file, rename_a_file, copy_file
def add_test_status(file_map):
list_map_filename = file_map["TEST"]
for map_filename in list_map_filename:
for filename, values in map_filename.items():
mapFileName = values[0]
map_test_status={}
id = mapFileName.attrib['id']
#test_state_name = "%s-%s.test"%(id, gethostname())
test_state_name = "%s.test"%(id)
test_state_path = os.path.join(settings.CACHE_JOB_PATH, test_state_name)
f= File(test_state_path)
try:
f.open(mode='r')
lines = f.readlines()
f.close()
for line in lines :
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_test_status[lineSplit[0].strip()] = lineSplit[1]
except OSError:
pass
values.append(map_test_status)
def get_test_state_file(fileid):
test_state_name = "%s.test"%(fileid)
test_state_path = os.path.join(settings.CACHE_JOB_PATH, test_state_name)
return test_state_path
def read_test_status(fileid):
map_test_status={}
test_state_file = get_test_state_file(fileid)
f= File(test_state_file)
try:
f.open(mode='r')
lines = f.readlines()
f.close()
for line in lines :
lineSplit = line.split("=")
if len(lineSplit) > 1:
map_test_status[lineSplit[0].strip()] = lineSplit[1].strip()
except OSError:
pass
return map_test_status
class TestFile :
def __init__(self,download_id) :
folder = Folder(os.path.join(settings.WORKSPACE_DOWNLOAD, get_workspace_dir_name(download_id) + downloader_django.common.constants.LISTING_MANUAL), create=True)
self._filetest = os.path.join(str(folder.getPath()), "listing.list.test")
def get_content_test_file(self,):
filename=""
f= File(self._filetest)
if f.exist():
f.open(mode='r')
else:
is_success = write_content_file(self._filetest, "")
f.open(mode='r')
filenames = f.readlines()
f.close()
return filenames
def save_content_test_file(self, test_filename, xml_filename):
is_success=False
f= File(self._filetest)
is_success = write_content_file(self._filetest, test_filename)
f.removeLockFileExist(settings.LOCK_FILE_EXTENSION)
if is_success:
downloadConfig = os.path.join(settings.CONF_PATH, xml_filename)
tmp_file= downloadConfig+ ".tmp"
is_success = rename_a_file(downloadConfig,tmp_file)
if is_success:
is_success = copy_file(tmp_file, downloadConfig)
remove_file(tmp_file)
return is_success
#
# -*- coding: UTF-8 -*-
#
"""XML File writer tools encapsulation
"""
__docformat__ = 'epytext'
from elementtree import ElementTree
import logging
from downloader_django.common.system.tasks import write_content_file
class XMLWriter(object):
"""
Class XMLWriter
"""
def __init__(self):
"""
XMLWriter initialization
"""
pass
def write(self, xmlTree, file):
is_success=False
try:
root=xmlTree.getroot()
self.__indent(root, 0)
ElementTree.dump(root)
xmlString = ElementTree.tostring(root,encoding='utf-8')
is_success = write_content_file(file, xmlString)
return is_success
except Exception:
#raise "Error during xml writing (file : %s)" % file
raise
def __indent(self, elem, level):
l = ["\n"]
for index in xrange(level):
l.append(" ")
i = ''.join(l)
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
self.__indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
def newXmlTree(self):
return ElementTree.ElementTree()
def xmlTreeSetRoot(self, tree, root):
tree._setroot(root)
def newNode(self, tag, text, tail="", attribute={}):
e = ElementTree.Element(tag, attrib=attribute)
e.text = text
e.tail = tail
return e
def newSubNode(self, node, tag, text, tail="", attribute={}):
e = ElementTree.SubElement(node, tag, attrib=attribute)
e.text = text
e.tail = tail
return e
def nodeSetText(self, node, text):
node.text = text
def nodeAddAttrib(self, node, key, value):
node.set(key, value)
This diff is collapsed.
from django.db import models
# Create your models here.
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
from django.conf.urls import patterns, include, url
urlpatterns = patterns('downloader_django.apps.monitor.views',
url( r'^$'
, 'test'
, name='root'),
url( r'^test$'
, 'test'
, name='test'),
url( r'^downloaders$'
, 'downloader_status'
, name='downloader_status'),
url( r'^downloaders_json$'
, 'downloader_status_json'
, name='downloader_status_json'),
url( r'^downloaders/lastdownloadfiles/(?P<download_id>\S+)$'
, 'last_download_files'
, name='last_download_files'),
url( r'^downloaders/lastdownloaddates/(?P<download_id>\S+)$'
, 'last_download_dates'
, name='last_download_dates'),
url( r'^downloaders/downloadforday/(?P<download_id>\S+)$'
, 'download_for_day'
, name='download_for_day'),
url( r'^downloaders/downloadbyday/(?P<download_id>\S+)$'
, 'download_by_day'
, name='download_by_day'),
url( r'^downloaders/stat$'
, 'download_stat'
, name='download_stat'),
url( r'^downloaders/stat/(?P<download_id>\S+)/details$'
, 'display_download_stat'
, name='display_download_stat'),
url( r'^downloaders/stat_json$'
, 'stat_json'
, name='stat_json'),
url( r'^downloaders/stat_json/(?P<download_id>\S+)$'
, 'display_download_stat_json'
, name='display_download_stat_json'),
)
import pkg_resources
pkg_resources.require("SQLAlchemy>=0.8.2")
from sqlalchemy import create_engine
from sqlalchemy import Table, Integer, String, MetaData, DateTime, Index
from sqlalchemy.orm import mapper, sessionmaker, relation, class_mapper
from sqlalchemy import func
from sqlalchemy import and_, or_
from downloader_django.common.constants import DATE_FORMAT_FRONT, DATE_FORMAT
from downloader_django.common.orderdict import odict
from downloader_django.common.downloaderUtil import get_workspace_dir_name
from collections import OrderedDict
import logging
import time, datetime
import sys, os
from django.conf import settings
from sqlalchemy.ext.declarative import declarative_base
from time import strftime
STATE_TO_DOWNLOAD=0
STATE_DOWNLOAD_ERROR=1
STATE_DOWNLOAD_SUCCESS=2
Base = declarative_base()
def init_downloader_database(download_id):
db_id = download_id
db_rel_path = 'downloader_%s.db'%(download_id)
db_prefix = 'sqlite:///'
download_wokspace=os.path.join(settings.WORKSPACE_DOWNLOAD, get_workspace_dir_name(download_id))
internal_path = os.path.join(download_wokspace, "internal")
dp = os.path.join(internal_path, db_rel_path)
db_url = db_prefix+dp
db = DownloaderDatabase(db_id, db_url)
db.create_engine()
return db
def map_class_to_some_table(cls, table, entity_name, **kw):
newcls = type(entity_name, (cls, ), {})
mapper(newcls, table, **kw)
return newcls
class DBDownload(object):
def __init__(self, id, filepath, state, ignition_date, last_update, sensingtime):
self.id = name
self.filepath = filepath
self.state = state
self.ignition_date=ignition_date
self.last_update=last_update
self.sensingtime=sensingtime
class DownloaderDatabase(object):
def __init__(self, download_id, database_path=None):
self.download_id = download_id
self.db = None
self.database_path = database_path
self.__downloadTable = None
self.metadata = None
self.__download_mapper = None
#self.engine=self.create_engine()
def get_session(self):
SessionMaker = sessionmaker(bind=self.db)
self.session = SessionMaker()
return self.session
def close_session(self):
self.session.close()
def create_engine(self):
self.db = create_engine('%s'%(self.database_path), echo=False)