Commit 410dc619 authored by MAISSIAT's avatar MAISSIAT
Browse files

Fix #67: rename the project to eo-dataflow-manager

parent 6d3e9ff4
...@@ -5,4 +5,4 @@ ...@@ -5,4 +5,4 @@
.pydevproject .pydevproject
.project .project
dist dist
downloader.egg-info eo_dataflow_manager.egg-info
#
#
#
#
# ------------------------------------------------------------------ # ------------------------------------------------------------------
FROM continuumio/miniconda3 FROM continuumio/miniconda3
...@@ -10,10 +6,10 @@ SHELL ["/bin/bash", "--login", "-c"] ...@@ -10,10 +6,10 @@ SHELL ["/bin/bash", "--login", "-c"]
ARG ROOTPATH=/tmp ARG ROOTPATH=/tmp
ARG WORKSPACE=$ROOTPATH/workspace ARG WORKSPACE=$ROOTPATH/workspace
ARG APPDATA=$ROOTPATH/appdata ARG APPDATA=$ROOTPATH/appdata
v
RUN cd $ROOTPATH && \ RUN cd $ROOTPATH && \
git clone https://gitlab.ifremer.fr/downloader/downloader_daemon.git && \ git clone https://gitlab.ifremer.fr/eo_dataflow_manager/eo_dataflow_manager.git && \
conda env create -f downloader_daemon/environment.yaml && \ conda env create -f eo_dataflow_manager/environment.yaml && \
apt-get -y update && \ apt-get -y update && \
apt-get -y install gcc && \ apt-get -y install gcc && \
conda init bash && \ conda init bash && \
...@@ -22,7 +18,7 @@ RUN cd $ROOTPATH && \ ...@@ -22,7 +18,7 @@ RUN cd $ROOTPATH && \
WORKDIR $ROOTPATH/downloader_daemon WORKDIR $ROOTPATH/downloader_daemon
RUN conda clean -a && \ RUN conda clean -a && \
conda activate downloader && \ conda activate eo_dataflow_manager && \
pip install -r requirements.txt --index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/pypi \ pip install -r requirements.txt --index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/pypi \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/simple \ --index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-public-snapshot/simple \ --extra-index-url https://nexus-test.ifremer.fr/repository/pypi-public-snapshot/simple \
...@@ -32,8 +28,6 @@ RUN conda clean -a && \ ...@@ -32,8 +28,6 @@ RUN conda clean -a && \
# ------------------------------------------------------------------ # ------------------------------------------------------------------
EXPOSE 80
ENV DATAPATH=/home ENV DATAPATH=/home
ENV CONFIG_FILE=${APPDATA}/application.yaml OPTIONS="-f" ENV CONFIG_FILE=${APPDATA}/application.yaml OPTIONS="-f"
...@@ -46,7 +40,7 @@ RUN mkdir -p ${WORKSPACE} && \ ...@@ -46,7 +40,7 @@ RUN mkdir -p ${WORKSPACE} && \
VOLUME [ "${WORKSPACE}" , "${APPDATA}" , "${DATAPATH}"] VOLUME [ "${WORKSPACE}" , "${APPDATA}" , "${DATAPATH}"]
ENTRYPOINT conda activate downloader && \ ENTRYPOINT conda activate eo_dataflow_manager && \
downloader -c ${CONFIG_FILE} ${OPTIONS} eo_dataflow_manager -c ${CONFIG_FILE} ${OPTIONS}
CMD [ "/bin/bash" ] CMD [ "/bin/bash" ]
# Downloader daemon # eo-dataflow-manager
Checks online services for changed in data files, and downloads them when needed. Checks online services for changed in data files, and downloads them when needed.
...@@ -31,7 +31,7 @@ extra-index-url= ...@@ -31,7 +31,7 @@ extra-index-url=
conda env create -f environment.yaml conda env create -f environment.yaml
# Activate created env # Activate created env
conda activate downloader conda activate eo-dataflow-manager
# Install the downloader # Install the downloader
./setup.py install ./setup.py install
......
...@@ -42,9 +42,9 @@ function init() ...@@ -42,9 +42,9 @@ function init()
else else
echo " => Initializing the ${DOWNLOADER_APPDATA} application data" echo " => Initializing the ${DOWNLOADER_APPDATA} application data"
fi fi
DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("downloader"); print(os.path.dirname(t.__file__))' 2> /dev/null` DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [[ $CONDA_PREFIX != "" ]]; then if [[ $CONDA_PREFIX != "" ]]; then
cp -r $CONDA_PREFIX/usr/share/downloader/* ${DOWNLOADER_APPDATA} ${OPTION} cp -r $CONDA_PREFIX/usr/share/eo-dataflow-manager/* ${DOWNLOADER_APPDATA} ${OPTION}
else else
cp -r ${DOWNLOADERPACKAGENAME}/conf/* ${DOWNLOADER_APPDATA} ${OPTION} cp -r ${DOWNLOADERPACKAGENAME}/conf/* ${DOWNLOADER_APPDATA} ${OPTION}
fi fi
...@@ -120,7 +120,7 @@ UPDATE="NO" ...@@ -120,7 +120,7 @@ UPDATE="NO"
parser $* parser $*
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show downloader | grep Version | cut -c 10-` export DOWNLOADER_VERSION=`pip --disable-pip-version-check show eo-dataflow-manager | grep Version | cut -c 10-`
echo "Downloader version = ${DOWNLOADER_VERSION}" echo "Downloader version = ${DOWNLOADER_VERSION}"
echo "Downloader appdata = ${DOWNLOADER_APPDATA}" echo "Downloader appdata = ${DOWNLOADER_APPDATA}"
...@@ -133,8 +133,8 @@ SCRIPT_DIR=$(dirname $(readlink -f "$0")) ...@@ -133,8 +133,8 @@ SCRIPT_DIR=$(dirname $(readlink -f "$0"))
export PYTHONPATH=${SCRIPT_DIR} export PYTHONPATH=${SCRIPT_DIR}
export DOWNLOADER_APPD=${DOWNLOADER_APPDATA}/downloader.appd export DOWNLOADER_APPD=${DOWNLOADER_APPDATA}/eo-dataflow-manager.appd
export DOWNLOADER_PATTERN="[p]ython.*Downloader.py --workspace_dir * --appdata_dir ${DOWNLOADER_APPDATA}" export DOWNLOADER_PATTERN="[p]ython.*eo_dataflow_manager.py --workspace_dir * --appdata_dir ${DOWNLOADER_APPDATA}"
#External constants #External constants
...@@ -176,7 +176,7 @@ fi ...@@ -176,7 +176,7 @@ fi
################################################################ ################################################################
### VERIFY DOWNLOADER INSTALLATION ### VERIFY DOWNLOADER INSTALLATION
################################################################ ################################################################
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("downloader"); print(os.path.dirname(t.__file__))' 2> /dev/null` export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [ -z ${DOWNLOADERPACKAGENAME} ]; then if [ -z ${DOWNLOADERPACKAGENAME} ]; then
echo "Error: downloader package is not installed, check the path to the active Python." echo "Error: downloader package is not installed, check the path to the active Python."
exit 1 exit 1
......
...@@ -129,7 +129,7 @@ UPDATE="NO" ...@@ -129,7 +129,7 @@ UPDATE="NO"
parser $* parser $*
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show downloader | grep Version | cut -c 10-` export DOWNLOADER_VERSION=`pip --disable-pip-version-check show eo-dataflow-manager | grep Version | cut -c 10-`
echo "Downloader version = ${DOWNLOADER_VERSION}" echo "Downloader version = ${DOWNLOADER_VERSION}"
echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}" echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}"
...@@ -141,9 +141,9 @@ echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}" ...@@ -141,9 +141,9 @@ echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}"
SCRIPT_DIR=$(dirname $(readlink -f "$0")) SCRIPT_DIR=$(dirname $(readlink -f "$0"))
export PYTHONPATH=${SCRIPT_DIR} export PYTHONPATH=${SCRIPT_DIR}
export DOWNLOADER_WRKS_FILE=${DOWNLOADER_WORKSPACE}/downloader.wrks export DOWNLOADER_WRKS_FILE=${DOWNLOADER_WORKSPACE}/eo-dataflow-manager.wrks
export DOWNLOADER_LOCK="${DOWNLOADER_WORKSPACE}/log/scheduler.log.lock" export DOWNLOADER_LOCK="${DOWNLOADER_WORKSPACE}/log/scheduler.log.lock"
export DOWNLOADER_PATTERN="[p]ython.*Downloader.py --workspace_dir ${DOWNLOADER_WORKSPACE}" export DOWNLOADER_PATTERN="[p]ython.*eo_dataflow_manager.py --workspace_dir ${DOWNLOADER_WORKSPACE}"
if [[ $HELP == "YES" ]]; then if [[ $HELP == "YES" ]]; then
usage usage
...@@ -178,7 +178,7 @@ fi ...@@ -178,7 +178,7 @@ fi
################################################################ ################################################################
### VERIFY DOWNLOADER INSTALLATION ### VERIFY DOWNLOADER INSTALLATION
################################################################ ################################################################
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("downloader"); print(os.path.dirname(t.__file__))' 2> /dev/null` export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [ -z ${DOWNLOADERPACKAGENAME} ]; then if [ -z ${DOWNLOADERPACKAGENAME} ]; then
echo "Error: downloader package is not installed, check the path to the active Python." echo "Error: downloader package is not installed, check the path to the active Python."
exit 1 exit 1
......
...@@ -12,9 +12,9 @@ ...@@ -12,9 +12,9 @@
# --- Folder paths # --- Folder paths
# paths: # paths:
# --- Workspace folder path # --- Workspace folder path
# workspace: /tmp/downloader/workspace # workspace: /tmp/eo-dataflow-manager/workspace
# --- App data and config folder # --- App data and config folder
# appdata: /srv/downloader/ # appdata: /srv/eo-dataflow-manager/
# --- Administrator contact list # --- Administrator contact list
# admins: [] # admins: []
...@@ -50,14 +50,14 @@ ...@@ -50,14 +50,14 @@
# queue_name: dl-emm # queue_name: dl-emm
# # --- job files config # # --- job files config
# jobs: jobs:
# --- List of output target for writing job files. Available targets: # --- List of output target for writing job files. Available targets:
# --- elasticsearch: Push emm jobs to an elasticsearch database # --- elasticsearch: Push emm jobs to an elasticsearch database
# --- rabbitmq: Push emm jobs to a RabbitMQ queue # --- rabbitmq: Push emm jobs to a RabbitMQ queue
# targets: [] # targets: []
# --- Elasticsearch configuration # --- Elasticsearch configuration
# elasticsearch: elasticsearch:
# hosts: [] hosts: []
# scheme: http # scheme: http
# user: null # user: null
# password: null # password: null
...@@ -85,7 +85,7 @@ ...@@ -85,7 +85,7 @@
# password: null # password: null
# virtual_host: "/" # virtual_host: "/"
# broker_heartbeat: 30 # broker_heartbeat: 30
# imports: [downloader.worker.celery_tasks] # imports: [eo-dataflow-manager.worker.celery_tasks]
# result_backend: amqp # result_backend: amqp
# result_expires: 300 # result_expires: 300
# task_serializer: json # task_serializer: json
......
*** Documenter le coup du timestamp qui est utilise dans la base comme curseur pour les rapports
*** Voir pour les filtres, l'impact de isfilereportable et isdirectoryreportable, au niveau de l'ordre (continue qui fait qu'on passera pas forcément dans l'autre filtre alors qu'il peut etre forcé dans le filtre suivant...)
*** creer le dossier auto dans var/lib a l'install (pour que les droits dessus soient corrects)
*** bug du relancement dans la meme seconde du dcheck : cela provoque une impossibilité de générer le rapport (probablement du au datetime...)
DEBUG:AbstractReport:fetchExecutionIdFromDatesLimits : date_from = 2007-04-27 16:45:15, date_to = 2007-04-27 16:45:18
WARNING:AbstractReport:isExecutionPeriodValid : fromExec.id >= toExec.id (12 >= 12)... no report !
AMELIORER :
*** faire une constconfigstring, et/ou mieux gerer les changements de config !
*** localpath : ameliorer le systeme de walk pour les dossier si possible. Cas d'exemple : le downloadMonitor
*** message d'erreur qd le _ftp_parse_listing n'est pas correct (cas d'un scan ftp.ifremer.fr/ifremer/cersat par exemple)
*** Faire en sorte de voir les timestamp lors du parcours des dossier...
*** performances de scan
*** Voir s'il faut rajouter une colonne path et une filename
*** Faire en sorte que les rapports renvoient des strings, pour pouvoir les integrer les uns dans les autres
*** visibilite des fichiers cachés ? Absent pour ftp... (peut etre meme pour localpath)
--> prise en compte par défaut ? option ?
*** logging.info envoie les infos sur stderr... c'est pas cool pour les cron!!!!
Module FTP :
*** mauvaise gestion des liens symboliques : ne les considère jamais comme répertoire... voir pour le follow symlinks
*** mettre une option sur le FORCE_CHROOT
\ No newline at end of file
#!/bin/sh
nosetests --with-doctest --doctest-tests --doctest-extension=.txt --doctest-extension=.py --exe $*
#!/bin/sh
cd dchecktools
./run_all_tests.sh
#!/usr/bin/env python
import os
import unittest
import doctest
import sys
from optparse import OptionParser
suite = unittest.TestSuite()
EXCLUDES = ['.svn']
def check_directory(directory):
sys_path_ori = sys.path
sys.path.insert(0, os.path.abspath(directory))
os.chdir(directory)
fnames = os.listdir(os.getcwd())
# print "on est dans ", os.getcwd() ,"les fichiers sont :", fnames
for filename in fnames:
if filename.endswith('.py'):
if filename.startswith('unittest_'):
mod = __import__(filename[:-3])
if 'suite' in dir(mod):
suite.addTest(mod.suite())
else:
suite.addTest(
doctest.DocFileSuite(
filename,
module_relative=False))
elif filename.endswith('.txt'):
suite.addTest(
doctest.DocFileSuite(
filename,
module_relative=False))
sys.path = sys_path_ori
def recursiv_check_directory(directory):
lstdir = sorted(os.listdir(directory))
check_directory(directory)
for file in lstdir:
curdir = os.getcwd()
if os.path.isdir(file):
if file not in EXCLUDES:
recursiv_check_directory(file)
os.chdir(curdir)
def run(directory, recursive=True, verbose=False):
# lstdir = os.listdir(directory)
# lstdir.sort()
# check_directory(directory)
if recursive:
recursiv_check_directory(directory)
else:
check_directory(directory)
# for file in lstdir:
# curdir = os.getcwd()
# if os.path.isdir(file):
# if not file in EXCLUDES:
# print "TESTING DIRECTORY : ", file
# check_directory(file)
# os.chdir(curdir)
test_res = None
if len(
sys.argv) > 1 and (
sys.argv[1] == '-v' or sys.argv[1] == '--verbose'):
test_res = unittest.TextTestRunner(verbosity=10).run(suite)
else:
test_res = unittest.TextTestRunner().run(suite)
return test_res.wasSuccessful()
def getOptions():
# Get a parser
usage = "usage: %prog [options] [path]"
parser = OptionParser(usage=usage)
# Set need argument
parser.add_option("-v", "--verbose",
action="store_true",
dest="verbose_mode", metavar="",
help="output more verbose",
default=False)
parser.add_option("-l", "--local",
action="store_true",
dest="local_mode", metavar="",
help="don't test in recursive mode",
default=False)
(options, args) = parser.parse_args()
return options, args
def main(argv=None):
if argv is None:
argv = sys.argv
options, args = getOptions()
verbose_mode = options.verbose_mode
recursive_mode = not options.local_mode
directory = '.'
if len(args) != 0:
directory = args[0]
# print "DIRECTORY = ", directory
success = run(directory, recursive=recursive_mode, verbose=verbose_mode)
if success:
return 0
else:
return 1
if __name__ == "__main__":
sys.exit(main())
name: downloader name: eo-dataflow-manager
channels: channels:
- default - default
- conda-forge - conda-forge
......
/__init__.pyc
/StringFilters.pyc
/DataReaderFactory.pyc
/__init__.pyc
from dchecktools.common.errors import DC_ConfigError from eo_dataflow_manager.dchecktools.common.errors import DC_ConfigError
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, DateTime, Boolean, and_, or_, desc, func, column from sqlalchemy import Column, Integer, String, DateTime, Boolean, and_, or_, desc, func, column
......
...@@ -7,7 +7,7 @@ __revision__ = '0.2' ...@@ -7,7 +7,7 @@ __revision__ = '0.2'
def getHumanSize(size): def getHumanSize(size):
""" """
Returns human readable size string from byte-size (returned by os.getsize() for example) Returns human readable size string from byte-size (returned by os.getsize() for example)
>>> from dchecktools.common.humansize import getHumanSize >>> from eo_dataflow_manager.dchecktools.common.humansize import getHumanSize
>>> getHumanSize(500) >>> getHumanSize(500)
'500.00' '500.00'
>>> getHumanSize(1024) >>> getHumanSize(1024)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment