Unverified Commit ca9ac663 authored by CHARLES's avatar CHARLES 🐧
Browse files

README instructions & autoformatting

parent 1e6c4baa
......@@ -18,3 +18,34 @@ conda activate downloader
# Install the downloader
./setup.py install
```
# Development
## Tooling
```
# Install developer tools
pip install -r requirements-dev.txt
```
### Linter
Using command-line:
```bash
# Run linter against current codebase
pylint --errors-only --output-format=colorized lib/
# The --errors-only flag only shows the errors
```
There are also plugins for many editors and IDEs.
### Code formatting
The code has been automatically formatted to follow PEP8 standard.
```bash
# Format every python source file in lib/ folder
autopep8 --in-place --recursive -j8 lib/
```
There are also plugins for many editors and IDEs.
\ No newline at end of file
......@@ -204,7 +204,8 @@ class SqliteBaseFilesInfos(object):
exec_infos_columns.append(Column("title", String(40)))
exec_infos_columns.append(Column("data", TEXT))
self.exec_infos = Table('exec_infos', self.metadata, *exec_infos_columns, autoload=autoload)
self.exec_infos = Table(
'exec_infos', self.metadata, *exec_infos_columns, autoload=autoload)
if not autoload:
try:
self.exec_infos.create(checkfirst=True)
......@@ -221,10 +222,13 @@ class SqliteBaseFilesInfos(object):
primary_key=True,
autoincrement=True))
executions_columns.append(Column("date_start", DateTime()))
executions_columns.append(Column("date_stop", DateTime(), nullable=True))
executions_columns.append(Column("valid_execution", Boolean, nullable=True))
executions_columns.append(
Column("date_stop", DateTime(), nullable=True))
executions_columns.append(
Column("valid_execution", Boolean, nullable=True))
self.executions = Table('executions', self.metadata, *executions_columns, autoload=autoload)
self.executions = Table(
'executions', self.metadata, *executions_columns, autoload=autoload)
if not autoload:
try:
self.executions.create(checkfirst=True)
......@@ -250,7 +254,8 @@ class SqliteBaseFilesInfos(object):
column = self.__getColumn(column_key)
files_columns.append(column)
self.infosKeyList += [column.name]
self.files = Table('files', self.metadata, *files_columns, autoload=autoload)
self.files = Table('files', self.metadata, *
files_columns, autoload=autoload)
if not autoload:
try:
self.files.create(checkfirst=True)
......@@ -274,7 +279,8 @@ class SqliteBaseFilesInfos(object):
mapper(Execution, self.executions)
self.current_execution = Execution()
self.current_execution.date_start = datetime.fromtimestamp(self.config._start_time)
self.current_execution.date_start = datetime.fromtimestamp(
self.config._start_time)
self.current_execution.date_stop = None
self.current_execution.valid_execution = None
self.session.bulk_save_objects([self.current_execution])
......@@ -329,7 +335,8 @@ class SqliteBaseFilesInfos(object):
elif column_key == 'gid':
return Column('gid', Integer)
else:
raise RuntimeError("Unknown column key : %s").with_traceback(column_key)
raise RuntimeError(
"Unknown column key : %s").with_traceback(column_key)
def getInfo(self, filename, infoKey):
# return select([self.files], self.files.c.filename==file).execute()
......
......@@ -63,7 +63,8 @@ def spliturl(url, defaultpath='/'):
'spliturl %s: scheme:%s username_password_domain:%s path:%s query:%s fragment:%s' %
(url, scheme, username_password_domain, path, query, fragment))
username_password, domain = urllib.parse.splituser(username_password_domain)
username_password, domain = urllib.parse.splituser(
username_password_domain)
if username_password:
username, password = urllib.parse.splitpasswd(username_password)
domain, port = urllib.parse.splitport(domain)
......
......@@ -148,7 +148,8 @@ def _ftp_parse_unix(listing):
continue
words = line.split(None, 8)
if len(words) < 6:
print('Warning: Error reading short line (unix) : ', line, file=sys.stderr)
print('Warning: Error reading short line (unix) : ',
line, file=sys.stderr)
continue
# Get the filename.
......@@ -229,7 +230,8 @@ def _ftp_parse_unix2(listing):
continue
words = line.split(None, 7)
if len(words) < 6:
print('Warning: Error reading short line (unix2) : ', line, file=sys.stderr)
print('Warning: Error reading short line (unix2) : ',
line, file=sys.stderr)
continue
# Get the filename.
......
......@@ -5,7 +5,9 @@ from dchecktools.protocols.AbstractProtocol import AbstractProtocol
from dchecktools.common.errors import DC_ConfigError, DC_ConnectionError
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.error
import urllib.parse
import logging
import socket
......@@ -105,7 +107,8 @@ class Protocol_http_jma_jra_gribfinal(AbstractProtocol):
self.__password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
self.__password_mgr.add_password(
None, self.server + '/' + self.path, self.username, self.password)
self.__auth_handler = urllib.request.HTTPBasicAuthHandler(self.__password_mgr)
self.__auth_handler = urllib.request.HTTPBasicAuthHandler(
self.__password_mgr)
self.__http_opener = urllib.request.build_opener(self.__auth_handler)
urllib.request.install_opener(self.__http_opener)
......
......@@ -6,7 +6,9 @@ from dchecktools.common.basefileinfos import File
from dchecktools.common.errors import DC_ConfigError, DC_ConnectionError
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.error
import urllib.parse
import logging
import os
import socket
......
......@@ -6,7 +6,9 @@ from dchecktools.common.basefileinfos import File
from dchecktools.common.errors import DC_ConfigError, DC_ConnectionError
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.error
import urllib.parse
import logging
import os
import socket
......
......@@ -6,7 +6,9 @@ from dchecktools.common.basefileinfos import File
from dchecktools.common.errors import DC_ConfigError, DC_ConnectionError
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.error
import urllib.parse
import logging
import os
import socket
......
......@@ -5,7 +5,9 @@ from dchecktools.protocols.AbstractProtocol import AbstractProtocol
from dchecktools.common.basefileinfos import File, FILE_URL_SEPARATOR
from dchecktools.common.errors import DC_ConfigError, DC_UrlError, DC_DbError
import urllib.request, urllib.error, urllib.parse
import urllib.request
import urllib.error
import urllib.parse
import logging
import socket
from xml.etree import ElementTree
......
......@@ -619,7 +619,8 @@ class Message(BaseElement):
xml_declaration=xml_declaration)
def isValid(self, messageValidator, error_details=False):
assert messageValidator is not None and isinstance(messageValidator, IMessageValidator)
assert messageValidator is not None and isinstance(
messageValidator, IMessageValidator)
return messageValidator.validate(self, error_details=error_details)
......
......@@ -129,20 +129,24 @@ class File(object):
if compressLevel < 1 or compressLevel > 9:
raise ValueError("compressLevel must be between 0 and 9")
else:
raise ValueError("bad `compress`, value must be in File.LIST_COMPRESS")
raise ValueError(
"bad `compress`, value must be in File.LIST_COMPRESS")
if compress == File.GZIP:
opener = GzipFile
validMode = ("r", "a", "w", "rb", "ab", "wb")
args = {'filename': self.getName(), 'mode': mode, 'compresslevel': compressLevel}
args = {'filename': self.getName(), 'mode': mode,
'compresslevel': compressLevel}
if compress == File.BZIP2:
opener = BZ2File
validMode = ("r", "a", "w")
args = {'filename': self.getName(), 'mode': mode, 'compresslevel': compressLevel}
args = {'filename': self.getName(), 'mode': mode,
'compresslevel': compressLevel}
# Use opener, validMode and moreArgs to valid and do the opening
if mode not in validMode:
raise ValueError("Invalid mode value (in relationship of the compress type ?)")
raise ValueError(
"Invalid mode value (in relationship of the compress type ?)")
self.__fd = opener(*[], **args)
# save the mode in __mode
self.__mode = mode
......@@ -562,12 +566,14 @@ class File(object):
lf = File(self.getName() + lockExtension)
expired = True
if lf.exist():
expired = datetime.datetime.now() - datetime.datetime.fromtimestamp(lf.getTime()) > datetime.timedelta(seconds=delay)
expired = datetime.datetime.now() - datetime.datetime.fromtimestamp(lf.getTime()
) > datetime.timedelta(seconds=delay)
return expired
def acquireLock(self, lockvalid, lockExtension=".lock"):
if self.__lockFile is not None:
raise RuntimeError("File is already locked with %s" % str(self.__lockFile))
raise RuntimeError("File is already locked with %s" %
str(self.__lockFile))
lf = File(self.getName() + lockExtension)
if lf.exist():
......@@ -596,7 +602,8 @@ class File(object):
def getLock(self):
if self.__lockFile is None:
raise RuntimeError("File hasn't lock file (use acquireLock to get one)")
raise RuntimeError(
"File hasn't lock file (use acquireLock to get one)")
return self.__lockFile
def unLock(self):
......
......@@ -222,7 +222,8 @@ class Folder(object):
entry = self.__filesContent[key]
if not entry:
self.__filesContent[key] = File.File(os.path.join(self.__path, key))
self.__filesContent[key] = File.File(
os.path.join(self.__path, key))
# Create a Folder object for the folder 'key'
def __makeFolder(self, key):
......@@ -234,7 +235,8 @@ class Folder(object):
entry = self.__foldersContent[key]
if not entry:
self.__foldersContent[key] = Folder(os.path.join(self.__path, key), self.__recursive)
self.__foldersContent[key] = Folder(
os.path.join(self.__path, key), self.__recursive)
# Properties about the recursive mode
def getRecursive(self):
......@@ -275,7 +277,8 @@ class Folder(object):
if not self.__recursive:
# OLD : first = os.walk(self.__path).next()
next(os.walk(self.__path))
iterator = [next(os.walk(self.__path)), ] # Take only the first one
# Take only the first one
iterator = [next(os.walk(self.__path)), ]
else:
iterator = os.walk(self.__path) # Store Iterator in iterator
......@@ -285,7 +288,8 @@ class Folder(object):
for f in lf:
yield os.path.join(pathFromCurrent, f), True # => is a file
for folder in lF:
yield os.path.join(pathFromCurrent, folder), False # => not a file
# => not a file
yield os.path.join(pathFromCurrent, folder), False
# define __repr__ use to get a textual presentation of the object with print
def __repr__(self):
......@@ -490,4 +494,3 @@ class Folder(object):
yield F
for f in self.__filesContent:
yield f
......@@ -403,7 +403,8 @@ class DCheckReportListingBuilder(IListingBuilder):
#dc = dcheck.DCheck()
#options, args = dc.getOptions(argsList)
#dc.run(options, args)
self.__process = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True, text=True)
self.__process = Popen(
cmd, stdout=PIPE, stderr=PIPE, close_fds=True, text=True)
stdout_lines, stderr_lines = self.__process.communicate()
......
......@@ -1502,7 +1502,8 @@ class Download(threading.Thread):
FILE_URL_SEPARATOR)) == 2:
local_file = self.__provider.getFile(
session,
downloaded_file.remoteFilepath.split(FILE_URL_SEPARATOR)[1],
downloaded_file.remoteFilepath.split(
FILE_URL_SEPARATOR)[1],
tmp_storage_path)
else:
local_file = self.__provider.getFile(
......@@ -2178,6 +2179,7 @@ class Download(threading.Thread):
# 06/02/2018 PMT#33 : ajout de storeSafeDirectoriesWaiting
def storeSafeDirectoriesWaiting(self):
""" Pooling du dossier FOLDER_WAITING_TO_COMPLETE (safe storage folder), pour deplacer des que possible
les dossiers qui sont complets (SAFE) dans le final storage folder """
......@@ -2831,7 +2833,8 @@ class Download(threading.Thread):
while exist_in_history:
i += 1
listingDstPath = os.path.join(
self.__getFolder(FOLDER_LISTINGS_HISTORY).getPath(),
self.__getFolder(
FOLDER_LISTINGS_HISTORY).getPath(),
"%s_" %
(i) +
listingFile.getName(
......
......@@ -315,7 +315,8 @@ class DownloaderDatabase(object):
if state is not None:
qfilter = and_(qfilter, DBDownload.state.in_(state))
if ignition_date is not None:
qfilter = and_(qfilter, DBDownload.ignition_date <= ignition_date)
qfilter = and_(
qfilter, DBDownload.ignition_date <= ignition_date)
res = query.filter(qfilter).all()
if res is not None and len(res) > 0:
assert len(res) == 1
......
......@@ -452,7 +452,8 @@ class Scheduler(object):
"A session of Scheduler for this %s, may be already run !" %
SystemControllerCst.SYSTEM_CONTROLLER_WORKSPACE_NAME)
# DO A MORE EXPLICIT MESSAGE ##############"
self._log.error("Check the lock file %s before restart the scheduler" % (logFile.getName() + self.__lockfileExtension))
self._log.error("Check the lock file %s before restart the scheduler" % (
logFile.getName() + self.__lockfileExtension))
self._log.error("!" * 100)
sys.exit(EXIT_LOCKED)
# No problem (ie no raise in readControllerConfig, and log file not locked) => change log Handler
......@@ -1205,4 +1206,4 @@ class Scheduler(object):
def scheduler_main():
scd = Scheduler()
scd.main()
\ No newline at end of file
scd.main()
......@@ -417,7 +417,8 @@ class SynchronisationLocalRemote(object):
cmd = ['dcheck'] + argsList
self.__log.info("Cmd %s" % cmd)
process = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True, text=True)
process = Popen(cmd, stdout=PIPE, stderr=PIPE,
close_fds=True, text=True)
stdout_lines, stderr_lines = process.communicate()
......
pylint
autopep8
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment