Commit 410dc619 authored by MAISSIAT's avatar MAISSIAT
Browse files

Fix #67: rename the project to eo-dataflow-manager

parent 6d3e9ff4
The humansize module
====================
>>> from dchecktools.common.humansize import getHumanSize
>>> from eo_dataflow_manager.dchecktools.common.humansize import getHumanSize
>>> getHumanSize(0)
'0.00'
>>> getHumanSize(1024)
......
......@@ -7,7 +7,7 @@ __revision__ = '0.2'
import urllib.request
import urllib.error
from urllib.parse import urlsplit
from dchecktools.common.errors import DC_ConfigError
from eo_dataflow_manager.dchecktools.common.errors import DC_ConfigError
import sys
import logging
......@@ -20,7 +20,7 @@ log.setLevel(logging.DEBUG)
def spliturl(url, defaultpath='/'):
"""
Returns url splitted, to get scheme, domain, path, port, username, password
>>> from dchecktools.common.spliturl import spliturl
>>> from eo_dataflow_manager.dchecktools.common.spliturl import spliturl
>>> spliturl('http://www.monsite.com')
('http', 'www.monsite.com', '/', None, None, None)
>>> spliturl('http://username:password@www.monsite.com:80/folder1/subfolder/file.html')
......
#!/usr/bin/env python
import unittest
from dchecktools.common import spliturl
from eo_dataflow_manager.dchecktools.common import spliturl
class testSplitUrl(unittest.TestCase):
......
......@@ -2,10 +2,10 @@ module workingdir
================
>>> from dchecktools.common.workingdir import getDatabasePathFromUrl
>>> from eo_dataflow_manager.dchecktools.common.workingdir import getDatabasePathFromUrl
* Pour les url de type localpath :
>>> import dchecktools.common.workingdir as workingdir
>>> import eo_dataflow_manager.dchecktools.common.workingdir as workingdir
>>> workingdir.DEFAULT_CONFIG_WORKING_DIR='/export/home/application_data/dchecktools'
>>> getDatabasePathFromUrl('/')
'/export/home/application_data/dchecktools/var/lib/auto/_.db'
......@@ -32,7 +32,7 @@ TODO : ATTENTION, CA NE MARCHE PAS, MAIS ENVOYER UNE ERREUR ?
* En changeant le DEFAULT_CONFIG_WORKING_DIR :
>>> import dchecktools.common.workingdir as workingdir
>>> import eo_dataflow_manager.dchecktools.common.workingdir as workingdir
>>> workingdir.DEFAULT_CONFIG_WORKING_DIR = '/tmp'
>>> getDatabasePathFromUrl('/home/user')
'/tmp/var/lib/auto/_home_user.db'
......
......@@ -3,27 +3,23 @@
"""Start DCheck (and end) here - read arguments, set global settings, etc."""
from dchecktools.common.basefileinfos import SqliteBaseFilesInfos
from dchecktools.common.spliturl import setConfigFromUrl
from dchecktools.common import workingdir
from dchecktools.protocols import localpath
from dchecktools.protocols import http_ostia
from dchecktools.protocols import http_navy_rainglobal
from dchecktools.protocols import http_jma_jra_gribfinal
from dchecktools.protocols import https_directorylist
from dchecktools.protocols import ftp
from dchecktools.protocols import lslr_file
from dchecktools.protocols import http_stdmet
from dchecktools.protocols import https_opensearch
from dchecktools.protocols import sftp
from dchecktools.protocols import webdav
from dchecktools.protocols import ftps
from dchecktools.protocols import file_extractor
from dchecktools.filters.StringFilters import FileFilter
from dchecktools.common.errors import DC_Error, DC_ConfigError, DC_DbError
from dchecktools.plugins import DataReaderFactory
from eo_dataflow_manager.dchecktools.common.basefileinfos import SqliteBaseFilesInfos
from eo_dataflow_manager.dchecktools.common.spliturl import setConfigFromUrl
from eo_dataflow_manager.dchecktools.common import workingdir
from eo_dataflow_manager.dchecktools.protocols import localpath
from eo_dataflow_manager.dchecktools.protocols import https_directorylist
from eo_dataflow_manager.dchecktools.protocols import ftp
from eo_dataflow_manager.dchecktools.protocols import lslr_file
from eo_dataflow_manager.dchecktools.protocols import https_opensearch
from eo_dataflow_manager.dchecktools.protocols import sftp
from eo_dataflow_manager.dchecktools.protocols import webdav
from eo_dataflow_manager.dchecktools.protocols import ftps
from eo_dataflow_manager.dchecktools.protocols import file_extractor
from eo_dataflow_manager.dchecktools.filters.StringFilters import FileFilter
from eo_dataflow_manager.dchecktools.common.errors import DC_Error, DC_ConfigError, DC_DbError
from eo_dataflow_manager.dchecktools.plugins import DataReaderFactory
import logging
from optparse import OptionParser
......@@ -37,7 +33,7 @@ log = logging.getLogger('dcheck')
log.setLevel(logging.INFO)
VERSION = "0.8b"
VERSION = "7.0.0"
DEFAULT_FTP_LISTING_TYPE = "unix"
......@@ -370,14 +366,6 @@ class DCheck(object):
kwargs["skipPermissions"] = True \
if option["SkipPermissions"].capitalize() == "True" else False
obj = ftp.Protocol_ftp(**kwargs)
elif protocolname == "http_ostia":
obj = http_ostia.Protocol_http_ostia()
elif protocolname == "http_stdmet":
obj = http_stdmet.Protocol_http_stdmet()
elif protocolname == "http_navy_rainglobal":
obj = http_navy_rainglobal.Protocol_http_navy_rainglobal()
elif protocolname == "http_jma_jra_gribfinal":
obj = http_jma_jra_gribfinal.Protocol_http_jma_jra_gribfinal()
elif protocolname == "lslr":
obj = lslr_file.Protocol_lslr_file()
elif protocolname == "https_opensearch":
......@@ -389,14 +377,14 @@ class DCheck(object):
elif protocolname == "ftps":
obj = ftps.Protocol_ftps(option)
#elif protocolname == "https_directorylist":
elif protocolname in ( "https", "http"):
elif protocolname in ("https", "http"):
classFileExtractor = https_directorylist.FileExtractor
redirector = None
if "Redirector" in option:
redirector = option["Redirector"]
if "FileExtractor" in option:
fileExtractor = "FileExtractor_" + option["FileExtractor"]
if hasattr(file_extractor, fileExtractor):
if hasattr(file_extractor, fileExtractor):
module = file_extractor
log.debug(f"FileExtractor : {option['FileExtractor']} found in regular module")
else:
......@@ -529,7 +517,8 @@ class DCheck(object):
if len(args) > 0:
url = args[0]
setConfigFromUrl(config, url)
if config.protocol is None:
setConfigFromUrl(config, url)
if options.debug_mode:
config.debug = True
......
......@@ -5,12 +5,12 @@ import os
from datetime import datetime
from optparse import OptionParser
import logging
from dchecktools.filters.StringFilters import FileFilter
from dchecktools.common.errors import DC_ConfigError
from dchecktools.reports import Report_ListFiles
from dchecktools.common import workingdir
from dchecktools.common.spliturl import setConfigFromUrl
from dchecktools.common.basefileinfos import SqliteBaseFilesInfos
from eo_dataflow_manager.dchecktools.filters.StringFilters import FileFilter
from eo_dataflow_manager.dchecktools.common.errors import DC_ConfigError
from eo_dataflow_manager.dchecktools.reports import Report_ListFiles
from eo_dataflow_manager.dchecktools.common import workingdir
from eo_dataflow_manager.dchecktools.common.spliturl import setConfigFromUrl
from eo_dataflow_manager.dchecktools.common.basefileinfos import SqliteBaseFilesInfos
"""Start DReport (and end) here - read arguments, set global settings, etc."""
logging.basicConfig()
......@@ -83,7 +83,7 @@ class Config:
modulepath + ".py", msg)
error = DC_ConfigError(error_str)
log.error(error)
raise Exception(error_str) # Pour integration dans le downloader
raise Exception(error_str) # Pour integration dans le eo-dataflow-manager
# sys.exit(1)
module_attr = list(module.__dict__.keys())
self.configfile = modulepath
......
......@@ -5,7 +5,7 @@
# fcad: adding DirNameBasedDataReader class
from dchecktools.plugins.DataReader.IDataReader import IDataReader
from eo_dataflow_manager.dchecktools.plugins.DataReader.IDataReader import IDataReader
import logging
import datetime
......
......@@ -5,7 +5,7 @@
# fcad: adding DirNameBasedDataReader class
from dchecktools.plugins.DataReader.IDataReader import IDataReader
from eo_dataflow_manager.dchecktools.plugins.DataReader.IDataReader import IDataReader
import logging
import datetime
......
......@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
from eccodes import *
from downloader.scheduler.sc.IDataReader import IDataReader
from eo_dataflow_manager.scheduler.sc.IDataReader import IDataReader
class GribDataReader(IDataReader):
......
......@@ -2,7 +2,7 @@
# -*- coding: UTF-8 -*-
#
from dchecktools.plugins.DataReader.IDataReader import IDataReader
from eo_dataflow_manager.dchecktools.plugins.DataReader.IDataReader import IDataReader
import logging
import datetime
......
......@@ -2,7 +2,7 @@
# -*- coding: UTF-8 -*-
#
from dchecktools.plugins.DataReader.IDataReader import IDataReader
from eo_dataflow_manager.dchecktools.plugins.DataReader.IDataReader import IDataReader
import logging
import datetime
......
......@@ -7,7 +7,7 @@
#
from downloader.scheduler.sc.IDataReader import IDataReader
from eo_dataflow_manager.scheduler.sc.IDataReader import IDataReader
import logging
import datetime
......
......@@ -4,7 +4,7 @@
#!/usr/bin/env python
from dchecktools.plugins.DataReader.IDataReader import IDataReader
from eo_dataflow_manager.dchecktools.plugins.DataReader.IDataReader import IDataReader
import logging
import datetime
......
......@@ -46,7 +46,7 @@ def DataReaderFactory(
# Try to import the modules "classname"
try:
tmpModule = my_import(
"dchecktools.plugins.DataReader." + classname)
"eo_dataflow_manager.dchecktools.plugins.DataReader." + classname)
#tmpModule = my_import("plugins.DataReader." + classname)
#tmpModule = my_import("DataReader." + classname)
except ImportError:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment