Unverified Commit 209e13fd authored by CHARLES's avatar CHARLES 🐧
Browse files

Fixed some unittests

parent a5fdaa68
......@@ -36,6 +36,22 @@ Using command-line:
# Run linter against current codebase
pylint --errors-only --output-format=colorized lib/
# The --errors-only flag only shows the errors
# Detect more issues:
pylint --disable=C0103,C0111,C0301,C0302,C0411,C1801,R0205,R0902,R0904,R0912,R0913,R0914,R0915 lib/
# C0103: snake_case convention
# C0111: missing doc
# C0301: line too long
# C0302: too many lines
# C0411: import order
# C1801: length != 0 as condition
# R0205: explicit object inheritance
# R0902: too many attributes
# R0904: too many methods
# R0912: too many branches
# R0913: too many arguments
# R0914: too many local variables
# R0915: too many instructions
```
There are also plugins for many editors and IDEs.
......
......@@ -70,7 +70,8 @@ def getFileFilterStatement(
ignoreFiles,
ignoreDirectories,
ignoreSymLinks,
defaultReturn=1):
defaultReturn=True):
assert isinstance(defaultReturn, bool)
file_filter_stmt = defaultReturn
if ignoreFiles:
file_filter_stmt = and_(file_filter_stmt, table.isDirectory == 1)
......@@ -566,18 +567,19 @@ class SqliteBaseFilesInfos(object):
ignoreSymLinks=False,
checkUnmodified=False):
# SELECT filename, size, mtime FROM (
# SELECT filename, id_execution, size, mtime, count() as nbr FROM files
# WHERE filename IN (
# SELECT filename FROM (
# SELECT filename, count() as nbr
# from files
# WHERE files.id_execution = 3 OR files.id_execution = 4
# GROUP BY files.filename
# ) WHERE nbr == 2
# SELECT filename, id_execution, size, mtime, count() as nbr FROM files
# WHERE filename IN (
# SELECT filename FROM (
# SELECT filename, count() as nbr
# from files
# WHERE (files.id_execution = 3 OR files.id_execution = 4)
# GROUP BY files.filename
# ) WHERE nbr == 2
# )
# AND id_execution = 3 or id_execution = 4
# GROUP BY filename, size, mtime
# )
# AND id_execution = 3 or id_execution = 4
# GROUP BY filename, size, mtime
# ) WHERE nbr = 1 and id_execution = 4
# WHERE nbr = 1 and id_execution = 4
# if checkInfosList is [], we cannot know if file is modified...
# returning []
......@@ -586,7 +588,7 @@ class SqliteBaseFilesInfos(object):
"req_ModifiedFiles : checkInfosList is empty... cannot check for modifications")
return []
s0InfosList = getInfosColumnList(self.files.c, checkInfosList)
# s0InfosList = getInfosColumnList(self.files.c, checkInfosList)
s0 = select([self.files.c.filename,
self.files.c.isDirectory,
self.files.c.isSymLink,
......@@ -597,7 +599,7 @@ class SqliteBaseFilesInfos(object):
correlate=False) # correlate=False : no FROM statement otherwise...
fileFilterStmt = getFileFilterStatement(
s0.c, ignoreFiles, ignoreDirectories, ignoreSymLinks, defaultReturn=1)
s0.c, ignoreFiles, ignoreDirectories, ignoreSymLinks, defaultReturn=True)
s1 = select([s0.c.filename], and_(s0.c.nbr == 2, fileFilterStmt))
s2InfosList = getInfosColumnList(self.files.c, checkInfosList)
......@@ -642,7 +644,7 @@ class SqliteBaseFilesInfos(object):
ignoreFiles,
ignoreDirectories,
ignoreSymLinks,
defaultReturn=1)
defaultReturn=True)
s0InfosList = getInfosColumnList(self.files.c, checkInfosList)
s0 = select([self.files.c.id_execution,
......@@ -688,7 +690,7 @@ class SqliteBaseFilesInfos(object):
ignoreFiles,
ignoreDirectories,
ignoreSymLinks,
defaultReturn=1)
defaultReturn=True)
s1InfosList = getInfosColumnList(self.files.c, checkInfosList)
s1 = select([self.files.c.filename.label('filepath'),
......
......@@ -14,7 +14,7 @@ class FileFilter(object):
based on reference regexp or string
"""
def __init__(self, forceRegexp=[], ignoreRegexp=[], ignoreOlderThan=None, ignoreNewerThan=None, interestingByDefault=True):
def __init__(self, forceRegexp=None, ignoreRegexp=None, ignoreOlderThan=None, ignoreNewerThan=None, interestingByDefault=True):
self.setForceInterestingRegex(forceRegexp)
self.setIgnoreRegex(ignoreRegexp)
......@@ -24,11 +24,11 @@ class FileFilter(object):
def setForceInterestingRegex(self, rgx):
rgxList = rgx if isinstance(rgx, list) else [rgx]
self.__forceRegexp = [re.compile(r) for r in rgxList]
self.__forceRegexp = [re.compile(r) for r in rgxList if r is not None]
def setIgnoreRegex(self, rgx):
rgxList = rgx if isinstance(rgx, list) else [rgx]
self.__ignoreRegexp = [re.compile(r) for r in rgxList]
self.__ignoreRegexp = [re.compile(r) for r in rgxList if r is not None]
def setIgnoreOlderThan(self, date):
self.__ignoreOlderThan = date
......@@ -58,12 +58,12 @@ class FileFilter(object):
if regexp.match(file):
return False
if (self.setIgnoreOlderThan is not None) and (mtime is not None) and (
datetime.fromtimestamp(mtime) < self.setIgnoreOlderThan):
if (self.__ignoreOlderThan is not None) and (mtime is not None) and (
datetime.fromtimestamp(mtime) < self.__ignoreOlderThan):
return False
if (self.setIgnoreNewerThan is not None) and (mtime is not None) and (
datetime.fromtimestamp(mtime) > self.setIgnoreNewerThan):
if (self.__ignoreNewerThan is not None) and (mtime is not None) and (
datetime.fromtimestamp(mtime) > self.__ignoreNewerThan):
return False
return self.__interestingByDefault
The StringFilters module
========================
>>> import sys
>>> sys.path.append('..')
>>> from dchecktools.filters.StringFilters import FileFilter, DirectoryFilter
>>> filter = FileFilter()
>>> filter.isInteresting("myfile.txt")
True
>>> filter.interestingByDefault = False
>>> filter.isInteresting("myfile.txt")
False
>>> filter.forceFiles = ["myfile.txt"]
>>> filter.isInteresting("myfile.txt")
True
>>> directories = [ '/tmp', '/home/user/.parameters', '/pub/GHRSST/L2P/AMSRE/2006/032' ]
Testing default result, without filter (default = true)
>>> dirFilter = DirectoryFilter()
>>> map(dirFilter.isInteresting, directories)
[True, True, True]
Testing default result, without filter (default = false)
>>> dirFilter.interestingByDefault = False
>>> map(dirFilter.isInteresting, directories)
[False, False, False]
Testing forceDirectory (interestingByDefault = False) : "/home/user/.parameters"
>>> dirFilter = DirectoryFilter()
>>> dirFilter.interestingByDefault = False
>>> dirFilter.forceDirectories = [ "/home/user/.parameters" ]
>>> map(dirFilter.isInteresting, directories)
[False, True, False]
Testing forceDirectoryRegexp : (interestingByDefault = False) : ".*AMSRE/\d+/\d+$"
>>> dirFilter = DirectoryFilter()
>>> dirFilter.interestingByDefault = False
>>> dirFilter.forceDirectoriesRegexp = [ '.*AMSRE/\d+/\d+$' ]
>>> map(dirFilter.isInteresting, directories)
[False, False, True]
Testing ignoreDirectoryRegexp : (interestingByDefault = True) : ".*AMSRE/\d+/\d+$"
>>> dirFilter = DirectoryFilter()
>>> dirFilter.ignoreDirectoriesRegexp = [ '.*AMSRE/\d+/\d+$' ]
>>> map(dirFilter.isInteresting, directories)
[True, True, False]
File mode changed from 100755 to 100644
File mode changed from 100755 to 100644
The basefileinfos module
========================
>>> import sys, time
>>> sys.path.append('../..')
>>> from dchecktools.common import basefileinfos
>>> from dchecktools.dcheck import Config
Configuration de la basefileinfo et de l'execution :
>>> config = Config()
>>> config._start_time = time.time()
>>> config.database_path = './tmp_database_doctest.db'
>>> bfi = basefileinfos.SqliteBaseFilesInfos()
>>> bfi.setConfig(config)
Creation de la base de donnees si elle n'existe pas (connexion a la base autrement)
>>> bfi.createEngine(db_must_exist=False)
>>> bfi.createTables(['size', 'mtime'], autoload=False)
Deroulement d'une execution avec ajout d'un fichier
>>> bfi.initExecution()
>>> bfi.addConstant('config', 'xml configuration string')
>>> file = basefileinfos.File()
>>> file.filename = "/tmp/foo/file1.txt"
>>> file.size = 1024
>>> bfi.addFile(file)
>>> bfi.setCurrentExecutionStopDate()
>>> bfi.setValidExecution(True)
>>> bfi.flushEngine()
>>> print bfi.req_ListFiles()
[(u'/tmp/foo/file1.txt',)]
>>> exec_id_1 = bfi.req_LastTerminatedExecutions()[0].id
Deroulement d'une autre execution, quelques instants plus tard, avec modification du fichier, et ajout d'un ature fichier
>>> time.sleep(1)
>>> config._start_time = time.time()
>>> bfi.setConfig(config)
>>> bfi.initExecution()
>>> bfi.addConstant('config', 'xml configuration string')
>>> file = basefileinfos.File()
>>> file.filename = "/tmp/foo/file1.txt"
>>> file.size = 2048
>>> bfi.addFile(file)
>>> file = basefileinfos.File()
>>> file.filename = "/tmp/foo/file2.txt"
>>> file.size = 10
>>> bfi.addFile(file)
>>> bfi.setCurrentExecutionStopDate()
>>> bfi.setValidExecution(True)
>>> bfi.flushEngine()
>>> print bfi.req_ListFiles()
[(u'/tmp/foo/file1.txt',), (u'/tmp/foo/file2.txt',)]
>>> exec_id_2 = bfi.req_LastTerminatedExecutions()[0].id
Affichage des différences entre les deux dernieres executions :
>>> bfi.req_CreatedFiles(exec_id_2, exec_id_1)
[(u'/tmp/foo/file2.txt',)]
>>> bfi.req_ModifiedFiles(exec_id_2, exec_id_1, checkInfosList=['mtime'])
[]
>>> bfi.req_ModifiedFiles(exec_id_2, exec_id_1, checkInfosList=['size'])
[(u'/tmp/foo/file1.txt', 2048)]
>>> bfi.req_RemovedFiles(exec_id_2, exec_id_1)
[]
>>> bfi.req_RemovedFiles(exec_id_1, exec_id_2)
[(u'/tmp/foo/file2.txt',)]
#!/usr/bin/env python
import unittest
from dchecktools.common.basefileinfos import SqliteBaseFilesInfos
......@@ -8,17 +6,13 @@ from dchecktools.dreport import Config
import os
import time
# just used to get a valid path for the test database
current_relative_path = os.path.split(__file__)[0]
class testBaseFilesInfos(unittest.TestCase):
def setUp(self):
self.config = Config()
self.config._start_time = time.time()
self.config.database_path = os.path.join(
current_relative_path, 'test_database1.db')
self.config.database_path = os.path.join(os.path.dirname(__file__), 'test_database1.db')
self.bfi = SqliteBaseFilesInfos()
self.bfi.setConfig(self.config)
self.id_from = 1
......@@ -104,49 +98,39 @@ class testBaseFilesInfos(unittest.TestCase):
# check avec infos size et mtime
check_infos = ['size', 'mtime']
results = self.requestNbrFiles(check_infos)
self.assertNbrFiles(results, 11, 8, 2, 3)
self.assertNbrFiles(results, unmodified=11, modified=8, removed=2, created=3)
# en ne prenant en compte que la size
check_infos = ['size']
results = self.requestNbrFiles(check_infos)
self.assertNbrFiles(results, 17, 2, 2, 3)
self.assertNbrFiles(results, unmodified=17, modified=2, removed=2, created=3)
# en ignorant les fichiers
check_infos = ['size', 'mtime']
results = self.requestNbrFiles(check_infos, ignoreFiles=True)
self.assertNbrFiles(results, 2, 5, 0, 1)
self.assertNbrFiles(results, unmodified=2, modified=5, removed=0, created=1)
# en ignorant les dossiers
results = self.requestNbrFiles(check_infos, ignoreDirectories=True)
self.assertNbrFiles(results, 9, 3, 2, 2)
self.assertNbrFiles(results, unmodified=9, modified=3, removed=2, created=2)
# en ignorant les liens
results = self.requestNbrFiles(check_infos, ignoreSymLinks=True)
self.assertNbrFiles(results, 10, 7, 2, 2)
self.assertNbrFiles(results, unmodified=10, modified=7, removed=2, created=2)
# en ignorant fichiers et dossiers
results = self.requestNbrFiles(
check_infos, ignoreFiles=True, ignoreDirectories=True)
self.assertNbrFiles(results, 0, 0, 0, 0)
self.assertNbrFiles(results, unmodified=0, modified=0, removed=0, created=0)
# en ignorant fichiers et symlinks
results = self.requestNbrFiles(
check_infos, ignoreFiles=True, ignoreSymLinks=True)
self.assertNbrFiles(results, 2, 4, 0, 0)
self.assertNbrFiles(results, unmodified=2, modified=4, removed=0, created=0)
# en ignorant dossiers et symlinks
results = self.requestNbrFiles(
check_infos,
ignoreDirectories=True,
ignoreSymLinks=True)
self.assertNbrFiles(results, 8, 3, 2, 2)
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(testBaseFilesInfos))
return suite
if __name__ == '__main__':
unittest.main()
self.assertNbrFiles(results, unmodified=8, modified=3, removed=2, created=2)
File mode changed from 100755 to 100644
The humansize module
====================
>>> from dchecktools.common.humansize import getHumanSize
>>> getHumanSize(0)
'0.00'
>>> getHumanSize(1024)
'1.00Ko'
>>> getHumanSize(1024**2)
'1.00Mo'
>>> getHumanSize(1024**3)
'1.00Go'
>>> getHumanSize(1024**4)
'1.00To'
>>> getHumanSize(1024**5)
'1.00Po'
>>> getHumanSize(1024**6)
'1024.00Po'
#!/usr/bin/env python
import unittest
from dchecktools.filters import StringFilters
......@@ -98,13 +96,3 @@ class testStringFilters(unittest.TestCase):
self.assertEqual(self.filter.isInteresting('standardfile.ext'), False)
self.assertEqual(self.filter.isInteresting('forcedfile.txt'), True)
self.assertEqual(self.filter.isInteresting('ignoredfile.txt'), False)
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(testStringFilters))
return suite
if __name__ == '__main__':
unittest.main()
#!/usr/bin/env python
import unittest
from dchecktools.reports import Report_ListFiles
......@@ -17,13 +15,3 @@ class testReportListFiles(unittest.TestCase):
def test_EmptyReport(self):
self.display_titles = True
# self.report.doReport()
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(testReportListFiles))
return suite
if __name__ == '__main__':
unittest.main()
#!/usr/bin/env python
import unittest
from dchecktools.common import spliturl
......@@ -16,9 +14,3 @@ class SplitUrlTest(unittest.TestCase):
('', '', '/tmp/toto', None, None, None))
for i in range(len(url_lst)):
self.assertEqual(spliturl.spliturl(url_lst[i]), url_results[i])
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(SplitUrlTest))
return suite
#!/usr/bin/env python
import unittest
from dchecktools.common import spliturl
......@@ -16,13 +14,3 @@ class testSplitUrl(unittest.TestCase):
('', '', '/tmp/toto', None, None, None))
for i in range(len(url_lst)):
self.assertEqual(spliturl.spliturl(url_lst[i]), url_results[i])
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(testSplitUrl))
return suite
if __name__ == '__main__':
unittest.main()
module workingdir
================
>>> from dchecktools.common.workingdir import getDatabasePathFromUrl
* Pour les url de type localpath :
>>> import dchecktools.common.workingdir as workingdir
>>> workingdir.DEFAULT_CONFIG_WORKING_DIR='/export/home/application_data/dchecktools'
>>> getDatabasePathFromUrl("/tmp")
'/export/home/application_data/dchecktools/var/lib/auto/_tmp.db'
>>> getDatabasePathFromUrl('/')
'/export/home/application_data/dchecktools/var/lib/auto/_.db'
* Pour les url de type ftp :
>>> getDatabasePathFromUrl("ftp://ftp.monsite.fr")
'/export/home/application_data/dchecktools/var/lib/auto/ftp:__ftp.monsite.fr.db'
>>> getDatabasePathFromUrl("ftp://ftp.monsite.fr/")
'/export/home/application_data/dchecktools/var/lib/auto/ftp:__ftp.monsite.fr.db'
>>> getDatabasePathFromUrl("ftp://ftp.monsite.fr/rep1/rep2/fichier.txt")
'/export/home/application_data/dchecktools/var/lib/auto/ftp:__ftp.monsite.fr_rep1_rep2_fichier.txt.db'
* Pour les url de type http :
>>> getDatabasePathFromUrl("http://www.monsite.fr/")
'/export/home/application_data/dchecktools/var/lib/auto/http:__www.monsite.fr.db'
>>> getDatabasePathFromUrl("http://www.monsite.fr/rep1/rep2")
'/export/home/application_data/dchecktools/var/lib/auto/http:__www.monsite.fr_rep1_rep2.db'
* Cas speciaux
TODO : mettre ce test ailleurs pour assert raise
# >>> getDatabasePathFromUrl('')
* En changeant le DEFAULT_CONFIG_WORKING_DIR :
>>> import dchecktools.common.workingdir as workingdir
>>> workingdir.DEFAULT_CONFIG_WORKING_DIR = '/tmp'
>>> getDatabasePathFromUrl('/home/user')
'/tmp/var/lib/auto/_home_user.db'
......@@ -568,11 +568,11 @@ class Download(threading.Thread):
try:
self.__current_mtime_download_conf_file = File(
self.__download_conf_file).getTime()
self._log.debug(" mtime of the download configuration file : %s"
% (self.__current_mtime_download_conf_file))
self._log.debug(" mtime of the download configuration file : %s",
(self.__current_mtime_download_conf_file))
except OSError as e:
self._log.warning(
" the download configuration file : %s doesn't exist" %
" the download configuration file : %s doesn't exist",
(self.__current_mtime_download_conf_file))
raise DownloaderException(e)
......@@ -583,7 +583,7 @@ class Download(threading.Thread):
xr = XMLReader(self._log.name)
self._log.debug(" --> read of the download configuration : %s" %
self._log.debug(" --> read of the download configuration : %s",
(self.__download_conf_file))
try:
......@@ -597,12 +597,12 @@ class Download(threading.Thread):
self.__download_conf_file)
raise DownloaderException(e)
except ParseError as e:
self._log.warning("Download_config file parse error %s " %
self._log.warning("Download_config file parse error %s ",
self.__download_conf_file)
raise IOError("invalid download_config file %s " %
self.__download_conf_file)
except Exception as e:
self._log.warning("Read download_config file error %s (%s)" % (
self._log.warning("Read download_config file error %s (%s)", (
self.__download_conf_file, str(e)))
raise IOError("invalid download_config file %s " %
self.__download_conf_file)
......@@ -2179,7 +2179,6 @@ class Download(threading.Thread):
# 06/02/2018 PMT#33 : ajout de storeSafeDirectoriesWaiting
def storeSafeDirectoriesWaiting(self):
""" Pooling du dossier FOLDER_WAITING_TO_COMPLETE (safe storage folder), pour deplacer des que possible
les dossiers qui sont complets (SAFE) dans le final storage folder """
......
......@@ -2,10 +2,9 @@ django<2
netCDF4~=1.4.0
kombu~=4.3.0
amqp~=2.4.0
billiard==3.5.0.3
celery==4.1.0
pytz==2017.3
simplejson==3.13.2
celery~=4.2.1
pytz==2018.9
simplejson~=3.16.0
SQLAlchemy~=1.2.0
vine~=1.2.0
ifr_lib~=1.2.0
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment