Commit 410dc619 authored by MAISSIAT's avatar MAISSIAT
Browse files

Fix #67: rename the project to eo-dataflow-manager

parent 6d3e9ff4
......@@ -5,4 +5,4 @@
.pydevproject
.project
dist
downloader.egg-info
eo_dataflow_manager.egg-info
#
#
#
#
# ------------------------------------------------------------------
FROM continuumio/miniconda3
......@@ -10,10 +6,10 @@ SHELL ["/bin/bash", "--login", "-c"]
ARG ROOTPATH=/tmp
ARG WORKSPACE=$ROOTPATH/workspace
ARG APPDATA=$ROOTPATH/appdata
v
RUN cd $ROOTPATH && \
git clone https://gitlab.ifremer.fr/downloader/downloader_daemon.git && \
conda env create -f downloader_daemon/environment.yaml && \
git clone https://gitlab.ifremer.fr/eo_dataflow_manager/eo_dataflow_manager.git && \
conda env create -f eo_dataflow_manager/environment.yaml && \
apt-get -y update && \
apt-get -y install gcc && \
conda init bash && \
......@@ -22,7 +18,7 @@ RUN cd $ROOTPATH && \
WORKDIR $ROOTPATH/downloader_daemon
RUN conda clean -a && \
conda activate downloader && \
conda activate eo_dataflow_manager && \
pip install -r requirements.txt --index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/pypi \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-public-snapshot/simple \
......@@ -32,8 +28,6 @@ RUN conda clean -a && \
# ------------------------------------------------------------------
EXPOSE 80
ENV DATAPATH=/home
ENV CONFIG_FILE=${APPDATA}/application.yaml OPTIONS="-f"
......@@ -46,7 +40,7 @@ RUN mkdir -p ${WORKSPACE} && \
VOLUME [ "${WORKSPACE}" , "${APPDATA}" , "${DATAPATH}"]
ENTRYPOINT conda activate downloader && \
downloader -c ${CONFIG_FILE} ${OPTIONS}
ENTRYPOINT conda activate eo_dataflow_manager && \
eo_dataflow_manager -c ${CONFIG_FILE} ${OPTIONS}
CMD [ "/bin/bash" ]
# Downloader daemon
# eo-dataflow-manager
Checks online services for changed in data files, and downloads them when needed.
......@@ -31,7 +31,7 @@ extra-index-url=
conda env create -f environment.yaml
# Activate created env
conda activate downloader
conda activate eo-dataflow-manager
# Install the downloader
./setup.py install
......
......@@ -42,9 +42,9 @@ function init()
else
echo " => Initializing the ${DOWNLOADER_APPDATA} application data"
fi
DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("downloader"); print(os.path.dirname(t.__file__))' 2> /dev/null`
DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [[ $CONDA_PREFIX != "" ]]; then
cp -r $CONDA_PREFIX/usr/share/downloader/* ${DOWNLOADER_APPDATA} ${OPTION}
cp -r $CONDA_PREFIX/usr/share/eo-dataflow-manager/* ${DOWNLOADER_APPDATA} ${OPTION}
else
cp -r ${DOWNLOADERPACKAGENAME}/conf/* ${DOWNLOADER_APPDATA} ${OPTION}
fi
......@@ -120,7 +120,7 @@ UPDATE="NO"
parser $*
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show downloader | grep Version | cut -c 10-`
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show eo-dataflow-manager | grep Version | cut -c 10-`
echo "Downloader version = ${DOWNLOADER_VERSION}"
echo "Downloader appdata = ${DOWNLOADER_APPDATA}"
......@@ -133,8 +133,8 @@ SCRIPT_DIR=$(dirname $(readlink -f "$0"))
export PYTHONPATH=${SCRIPT_DIR}
export DOWNLOADER_APPD=${DOWNLOADER_APPDATA}/downloader.appd
export DOWNLOADER_PATTERN="[p]ython.*Downloader.py --workspace_dir * --appdata_dir ${DOWNLOADER_APPDATA}"
export DOWNLOADER_APPD=${DOWNLOADER_APPDATA}/eo-dataflow-manager.appd
export DOWNLOADER_PATTERN="[p]ython.*eo_dataflow_manager.py --workspace_dir * --appdata_dir ${DOWNLOADER_APPDATA}"
#External constants
......@@ -176,7 +176,7 @@ fi
################################################################
### VERIFY DOWNLOADER INSTALLATION
################################################################
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("downloader"); print(os.path.dirname(t.__file__))' 2> /dev/null`
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [ -z ${DOWNLOADERPACKAGENAME} ]; then
echo "Error: downloader package is not installed, check the path to the active Python."
exit 1
......
......@@ -129,7 +129,7 @@ UPDATE="NO"
parser $*
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show downloader | grep Version | cut -c 10-`
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show eo-dataflow-manager | grep Version | cut -c 10-`
echo "Downloader version = ${DOWNLOADER_VERSION}"
echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}"
......@@ -141,9 +141,9 @@ echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}"
SCRIPT_DIR=$(dirname $(readlink -f "$0"))
export PYTHONPATH=${SCRIPT_DIR}
export DOWNLOADER_WRKS_FILE=${DOWNLOADER_WORKSPACE}/downloader.wrks
export DOWNLOADER_WRKS_FILE=${DOWNLOADER_WORKSPACE}/eo-dataflow-manager.wrks
export DOWNLOADER_LOCK="${DOWNLOADER_WORKSPACE}/log/scheduler.log.lock"
export DOWNLOADER_PATTERN="[p]ython.*Downloader.py --workspace_dir ${DOWNLOADER_WORKSPACE}"
export DOWNLOADER_PATTERN="[p]ython.*eo_dataflow_manager.py --workspace_dir ${DOWNLOADER_WORKSPACE}"
if [[ $HELP == "YES" ]]; then
usage
......@@ -178,7 +178,7 @@ fi
################################################################
### VERIFY DOWNLOADER INSTALLATION
################################################################
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("downloader"); print(os.path.dirname(t.__file__))' 2> /dev/null`
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [ -z ${DOWNLOADERPACKAGENAME} ]; then
echo "Error: downloader package is not installed, check the path to the active Python."
exit 1
......
......@@ -12,9 +12,9 @@
# --- Folder paths
# paths:
# --- Workspace folder path
# workspace: /tmp/downloader/workspace
# workspace: /tmp/eo-dataflow-manager/workspace
# --- App data and config folder
# appdata: /srv/downloader/
# appdata: /srv/eo-dataflow-manager/
# --- Administrator contact list
# admins: []
......@@ -50,14 +50,14 @@
# queue_name: dl-emm
# # --- job files config
# jobs:
jobs:
# --- List of output target for writing job files. Available targets:
# --- elasticsearch: Push emm jobs to an elasticsearch database
# --- rabbitmq: Push emm jobs to a RabbitMQ queue
# targets: []
# --- Elasticsearch configuration
# elasticsearch:
# hosts: []
elasticsearch:
hosts: []
# scheme: http
# user: null
# password: null
......@@ -85,7 +85,7 @@
# password: null
# virtual_host: "/"
# broker_heartbeat: 30
# imports: [downloader.worker.celery_tasks]
# imports: [eo-dataflow-manager.worker.celery_tasks]
# result_backend: amqp
# result_expires: 300
# task_serializer: json
......
*** Documenter le coup du timestamp qui est utilise dans la base comme curseur pour les rapports
*** Voir pour les filtres, l'impact de isfilereportable et isdirectoryreportable, au niveau de l'ordre (continue qui fait qu'on passera pas forcément dans l'autre filtre alors qu'il peut etre forcé dans le filtre suivant...)
*** creer le dossier auto dans var/lib a l'install (pour que les droits dessus soient corrects)
*** bug du relancement dans la meme seconde du dcheck : cela provoque une impossibilité de générer le rapport (probablement du au datetime...)
DEBUG:AbstractReport:fetchExecutionIdFromDatesLimits : date_from = 2007-04-27 16:45:15, date_to = 2007-04-27 16:45:18
WARNING:AbstractReport:isExecutionPeriodValid : fromExec.id >= toExec.id (12 >= 12)... no report !
AMELIORER :
*** faire une constconfigstring, et/ou mieux gerer les changements de config !
*** localpath : ameliorer le systeme de walk pour les dossier si possible. Cas d'exemple : le downloadMonitor
*** message d'erreur qd le _ftp_parse_listing n'est pas correct (cas d'un scan ftp.ifremer.fr/ifremer/cersat par exemple)
*** Faire en sorte de voir les timestamp lors du parcours des dossier...
*** performances de scan
*** Voir s'il faut rajouter une colonne path et une filename
*** Faire en sorte que les rapports renvoient des strings, pour pouvoir les integrer les uns dans les autres
*** visibilite des fichiers cachés ? Absent pour ftp... (peut etre meme pour localpath)
--> prise en compte par défaut ? option ?
*** logging.info envoie les infos sur stderr... c'est pas cool pour les cron!!!!
Module FTP :
*** mauvaise gestion des liens symboliques : ne les considère jamais comme répertoire... voir pour le follow symlinks
*** mettre une option sur le FORCE_CHROOT
\ No newline at end of file
#!/bin/sh
nosetests --with-doctest --doctest-tests --doctest-extension=.txt --doctest-extension=.py --exe $*
#!/bin/sh
cd dchecktools
./run_all_tests.sh
#!/usr/bin/env python
import os
import unittest
import doctest
import sys
from optparse import OptionParser
suite = unittest.TestSuite()
EXCLUDES = ['.svn']
def check_directory(directory):
sys_path_ori = sys.path
sys.path.insert(0, os.path.abspath(directory))
os.chdir(directory)
fnames = os.listdir(os.getcwd())
# print "on est dans ", os.getcwd() ,"les fichiers sont :", fnames
for filename in fnames:
if filename.endswith('.py'):
if filename.startswith('unittest_'):
mod = __import__(filename[:-3])
if 'suite' in dir(mod):
suite.addTest(mod.suite())
else:
suite.addTest(
doctest.DocFileSuite(
filename,
module_relative=False))
elif filename.endswith('.txt'):
suite.addTest(
doctest.DocFileSuite(
filename,
module_relative=False))
sys.path = sys_path_ori
def recursiv_check_directory(directory):
lstdir = sorted(os.listdir(directory))
check_directory(directory)
for file in lstdir:
curdir = os.getcwd()
if os.path.isdir(file):
if file not in EXCLUDES:
recursiv_check_directory(file)
os.chdir(curdir)
def run(directory, recursive=True, verbose=False):
# lstdir = os.listdir(directory)
# lstdir.sort()
# check_directory(directory)
if recursive:
recursiv_check_directory(directory)
else:
check_directory(directory)
# for file in lstdir:
# curdir = os.getcwd()
# if os.path.isdir(file):
# if not file in EXCLUDES:
# print "TESTING DIRECTORY : ", file
# check_directory(file)
# os.chdir(curdir)
test_res = None
if len(
sys.argv) > 1 and (
sys.argv[1] == '-v' or sys.argv[1] == '--verbose'):
test_res = unittest.TextTestRunner(verbosity=10).run(suite)
else:
test_res = unittest.TextTestRunner().run(suite)
return test_res.wasSuccessful()
def getOptions():
# Get a parser
usage = "usage: %prog [options] [path]"
parser = OptionParser(usage=usage)
# Set need argument
parser.add_option("-v", "--verbose",
action="store_true",
dest="verbose_mode", metavar="",
help="output more verbose",
default=False)
parser.add_option("-l", "--local",
action="store_true",
dest="local_mode", metavar="",
help="don't test in recursive mode",
default=False)
(options, args) = parser.parse_args()
return options, args
def main(argv=None):
if argv is None:
argv = sys.argv
options, args = getOptions()
verbose_mode = options.verbose_mode
recursive_mode = not options.local_mode
directory = '.'
if len(args) != 0:
directory = args[0]
# print "DIRECTORY = ", directory
success = run(directory, recursive=recursive_mode, verbose=verbose_mode)
if success:
return 0
else:
return 1
if __name__ == "__main__":
sys.exit(main())
name: downloader
name: eo-dataflow-manager
channels:
- default
- conda-forge
......
/__init__.pyc
/StringFilters.pyc
/DataReaderFactory.pyc
/__init__.pyc
from dchecktools.common.errors import DC_ConfigError
from eo_dataflow_manager.dchecktools.common.errors import DC_ConfigError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, DateTime, Boolean, and_, or_, desc, func, column
......
......@@ -7,7 +7,7 @@ __revision__ = '0.2'
def getHumanSize(size):
"""
Returns human readable size string from byte-size (returned by os.getsize() for example)
>>> from dchecktools.common.humansize import getHumanSize
>>> from eo_dataflow_manager.dchecktools.common.humansize import getHumanSize
>>> getHumanSize(500)
'500.00'
>>> getHumanSize(1024)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment