Commit 20917a00 authored by MAGHOUZ's avatar MAGHOUZ
Browse files

#94 including ifr_lib classes directly in downloader

parent 3f732528
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2018 Ifremer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
eo_dataflow_manager.ifr_lib_modules.ifr_collections
~~~~~~~~~~~~~~~~~~~~~~~
Provides utilities for managing collections.
"""
from collections import Counter, OrderedDict
try:
from collections.abc import Mapping
except:
from collections import Mapping
from eo_dataflow_manager.ifr_lib_modules import ifr_misc
import sys
def merge_dict(dict1, dict2, add_keys=True, __stack=None):
""" Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
updating only top-level keys, dict_merge recurses down into dicts nested
to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
``dct``.
This version will return a copy of the dictionary and leave the original
arguments untouched.
The optional argument ``add_keys``, determines whether keys which are
present in ``merge_dict`` but not ``dct`` should be included in the
new dict.
Types are conserved during merging, with the exception of None values. a
None value in dict1 can be overridden by any type, and a None value in
dict2 is skipped.
Args:
dict1 (dict) onto which the merge is executed
dict2 (dict): dct merged into dct
add_keys (bool): whether to add new keys
Returns:
dict: updated dict
"""
if dict2 is None:
return dict1
resultDict = dict1.copy()
if not add_keys:
dict2 = {
k: dict2[k]
for k in set(resultDict).intersection(set(dict2))
}
for k, v in dict2.items():
if (k in resultDict and isinstance(resultDict[k], dict)
and isinstance(dict2[k], Mapping)):
# Recursively merge
stack = (__stack + [k]) if __stack is not None else [k]
resultDict[k] = merge_dict(resultDict[k], dict2[k], add_keys=add_keys, __stack=stack)
else:
if k not in resultDict or resultDict[k] is None or isinstance(dict2[k], type(resultDict[k])):
# add or overwrite key/value
resultDict[k] = dict2[k]
elif v is not None:
stack = (__stack + [k]) if __stack is not None else [k]
raise Exception("Type mismatch while merging key %s: trying to overwrite %s with %s" % (".".join(stack), type(resultDict[k]), type(dict2[k])))
return resultDict
def to_dict(key, value, split_separator='.'):
"""Transforms a key/value pair to dict
Args:
key (str): a simple key or a namespace
value (str): the value of the leaf element
split_separator (str): the separator of items in the key
Returns:
dict: a dictionary
Examples:
>>> print(to_dict("toto", 'tutu'))
{'toto': 'tutu'}
>>> print(to_dict("toto.tata.titi", 'tutu'))
{'toto': {'tata': {'titi': 'tutu'}}}
"""
if split_separator not in key:
return {key: value}
def walk_items(items):
d = dict()
key = items.pop(0)
if len(items) > 0:
d[key] = walk_items(items)
else:
d[key] = value
return d
return walk_items(key.split(split_separator))
def dict_to_map(dictionary):
assert isinstance(dictionary, (dict, Mapping))
def process(value, path=None, dict_as_map=None):
if dict_as_map is None:
dict_as_map = dict()
if isinstance(value, (dict, Mapping)):
for key, value in value.items():
dict_as_map.update(
process(value, ("" if path is None else (path + ".")) + key, dict_as_map)
)
elif isinstance(value, list):
for i in range(0, len(value)):
dict_as_map.update(process(value[i], ("" if path is None else (path + ".")) + str(i), dict_as_map))
else:
return [(path, value)]
return dict_as_map
return process(dictionary)
def list_difference(list1, list2):
"""Checks elements from list1 not present in list2.
Args:
list1 (list): the main list
list2 (list): the list to compare content
Returns:
list: a list of missing items in list2
Examples:
>>> list1 = ["donald", "daisy", "mickey"]
>>> list2 = ["daisy", "mickey", "minnie"]
>>> print(list_difference(list1, list2))
['donald']
>>> print(list_difference(list2, list1))
['minnie']
"""
return list(set(list1).difference(list2))
def list_equals(list1, list2):
"""Checks two lists are equals
Args:
list1 (list): the first list
list2 (list): the second list
Returns:
bool: True if liste are the same, otherwise False
Examples:
>>> list1 = ["donald", "daisy", "mickey"]
>>> list2 = ["daisy", "mickey", "minnie"]
>>> print(list_equals(list1, list2))
False
>>> list3 = ["mickey", "donald", "daisy", "mickey"]
>>> print(list_equals(list1, list3))
True
"""
return set(list1) == set(list2)
def prettify_dict(dictionary,
item_prefix='- ',
key_value_separator=': ',
title=None,
header=('-' * 70),
footer=('-' * 70),
title_footer=('-' * 70),
item_sorted=False,
color_options=None
):
"""Converts a dictionary to a list of string representation.
Args:
dictionary (dict): a dictionary
title (str, optional): a title to display before the list of elements
item_prefix (str, optional): a prefix before each item. Default to '- '
key_value_separator (str, optional): a separator between key and value. Default to ': '
header (str, optional): a separator before the title.
footer (str, optional): a separator before the title.
title_footer (str, optional): a separator after the title.
item_sorted (bool, optional): if true, sort the items by the key, otherwise keep the default sort
color_options (str, optional): color options (ex: 'blue', 'bold_blue,bg_white', ...)
Returns:
list<str>: a prettified representation of the dictionary
Examples:
>>> dictionary = {"key1": "value1", "key3": "value3", "key2": "value2"}
>>> print(prettify_dict(dictionary))
[
'- key1: value1',
'- key3: value3',
'- key2: value2'
]
>>> print(prettify_dict(dictionary, title="My dict", item_sorted=True))
[
'----------------------------------------------------------------------',
'My dict',
'----------------------------------------------------------------------',
'- key1: value1', '
- key2: value2',
'- key3: value3'
]
"""
def colorize_item(item):
return ifr_misc.colorize(item, color_options)
def build_message(key, value):
return colorize_item('{}{}{}{}'.format(item_prefix, key, key_value_separator, value))
msg = list()
if header:
msg.append(colorize_item(header))
if title:
msg.append(colorize_item('{}'.format(title)))
if title_footer:
msg.append(colorize_item(title_footer))
if not dictionary:
return msg
if item_sorted:
for key, value in sorted(dictionary.items()):
msg.append(build_message(key, value))
else:
for key, value in dictionary.items():
msg.append(build_message(key, value))
if footer:
msg.append(colorize_item(footer))
return msg
def dict_to_str(dictionary,
line_separator='\n',
item_prefix='- ',
key_value_separator=' : ',
title=None,
header=('-' * 70),
footer=('-' * 70),
title_footer=('-' * 70),
item_sorted=False,
color_options=None
):
"""Converts a dictionary to a list of string representation.
Args:
dictionary (dict): a dictionary
line_separator (str): a line separator. Defaults to "\\n"
title (str, optional): a title to display before the list of elements
item_prefix (str, optional): a prefix before each item. Default to "- "
key_value_separator (str, optional): a separator between key and value.
Default to ": "
header (str, optional): a separator before the title.
footer (str, optional): a separator before the title.
title_footer (str, optional): a separator after the title.
item_sorted (bool, optional): if true, sort the items by the key,
otherwise keep the default sort
color_options (str, optional): color options (ex: 'blue', 'bold_blue,bg_white', ...)
Returns:
list<str>: a prettified representation of the dictionary
Examples:
>>> dictionary = {"key1": "value1", "key3": "value3", "key2": "value2"}
>>> print(dict_to_str(dictionary))
- key1: value1
- key3: value3
- key2: value2
>>> print(dict_to_str(dictionary, title="My dict", item_sorted=True))
----------------------------------------------------------------------
My dict
----------------------------------------------------------------------
- key1: value1
- key2: value2
- key3: value3
"""
return line_separator.join(prettify_dict(
dictionary=dictionary,
item_prefix=item_prefix,
key_value_separator=key_value_separator,
title=title,
header=header,
footer=footer,
title_footer=title_footer,
item_sorted=item_sorted,
color_options=color_options
))
def counter(list_of_elements):
"""Count the appearance number of each item in a list.
Args:
list_of_elements (list): a list of elements
Returns:
Counter: a dictionary of <element, number of occurrences>
Examples:
>>> dictionary = ['lorem.ipsum.key1', 'lorem.ipsum.key1', 'lorem.ipsum.key2']
>>> print(counter(dictionary))
Counter({'lorem.ipsum.key1': 2, 'lorem.ipsum.key2': 1})
>>> print(dict_to_str(counter(dictionary), title='Number of occurrences'))
----------------------------------------------------------------------
Number of occurrences
----------------------------------------------------------------------
- lorem.ipsum.key1: 2
- lorem.ipsum.key2: 1
"""
return Counter(list_of_elements)
def dict_to_ini(dictionary):
"""Transforms a dictionary to INI format
Args:
dictionary (dict): dictionary to transform
Returns:
str: the dictionary to INI format
Example:
>>> print(dict_to_ini({"key1": {"key3": "value3", "key2": "value2"}}))
[key1]
key3 = value3
key2 = value2
>>> print(dict_to_ini({"key1": {"key3": "value3", "key2": {"key4": "value"}, "key5": "value"}}))
[key1]
key3 = value3
key5 = value
[key1.key2]
key4 = value
"""
def dict_to_sections(dictionary, current_section=None, sections=None):
if sections is None:
sections = OrderedDict()
for key, value in dictionary.items():
if isinstance(value, dict):
new_section = (current_section + "." if current_section is not None else '') + key
sections[new_section] = OrderedDict()
dict_to_sections(value, new_section, sections)
else:
if value is None:
value = ''
if current_section is not None:
sections[current_section][key] = value
else:
if '' not in sections:
sections[''] = OrderedDict()
sections[''][key] = value
return sections
def sections_to_ini(sections):
result = list()
for section in sections.keys():
if sections[section] is None or len(sections[section]) == 0:
continue
if section != '':
result.append('[{}]'.format(section))
for key, value in sections[section].items():
result.append('{} = {}'.format(key, value))
result.append('')
# remove last ne line
result.pop()
# return list as string
return "\n".join(result)
sections = dict_to_sections(dictionary)
if sections is None or len(sections) == 0:
return None
return sections_to_ini(sections)
def ini_to_dict(ini_content):
"""Deserialize an INI content to dictionary
Only for python versions 3.2 and above.
Args:
ini_content (str): INI content
Returns:
dict: the dictionary
Example:
>>> print(ini_to_dict(dict_to_ini({"key1": {"key3": "value3", "key2": "value2"}})))
{"key1": {"key3": "value3", "key2": "value2"}}
>>> print(dict_to_ini({"key1": {"key3": "value3", "key2": {"key4": "value"}, "key5": "value"}}))
{"key1": {"key3": "value3", "key5": "value", "key2": {"key4": "value"}}}
"""
if sys.version_info < (3, 2):
return None
from eo_dataflow_manager.ifr_lib_modules.ifr_yaml import YamlConfig
from configparser import ConfigParser
parser = ConfigParser(interpolation=None)
parser.read_string(ini_content)
result = YamlConfig()
for section in parser.sections():
result[section] = OrderedDict()
for key, val in parser.items(section):
result[section][key] = val
return result.as_dict()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2018 Ifremer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
eo_dataflow_manager.ifr_lib_modules.ifr_datetimes
~~~~~~~~~~~~~~~~~~~~~
Provides some utilities for managing date and time.
"""
import sys
from calendar import monthrange
import datetime
import dateparser
import time
from eo_dataflow_manager.ifr_lib_modules.ifr_exception import ConversionException, BadParameterException
from eo_dataflow_manager.ifr_lib_modules import ifr_numbers
if sys.version_info < (3, 2):
DEFAULT_ISO_FORMAT = '%Y-%m-%dT%H:%M:%S'
else:
DEFAULT_ISO_FORMAT = '%Y-%m-%dT%H:%M:%S%z'
def now():
"""Retrieves the current datetime.
Returns:
datetime.datetime: The current datetime object
"""
return datetime.datetime.now()
def elapsed_time(date1, date2):
"""Calculates the elapsed time between the two dates
Args:
date1 (datetime.datetime): a date
date2 (datetime.datetime): a date
Returns:
datetime.timedelta: The elapsed time between date2 and date1
Examples:
>>> date1 = str_to_datetime('2018-02-01T14:28:42+0200')
>>> date2 = str_to_datetime('2018-08-31T07:48:12+0200')
>>> print(elapsed_time(date1, date2))
210 days, 17:19:30
"""
return date2 - date1
def datetime_to_str(date=now(), pattern='%d/%m/%Y'):
"""Converts a datetime to a str.
Args:
date (datetime.datetime): The datetime object to convert. Defaults to now()
pattern: The string representation pattern. Defaults to '%d/%m/%Y'.
Returns:
str: The datetime object converted to string
Raises:
ConversionException: if conversion failed (bad pattern, ...)
Examples:
>>> date = str_to_datetime('2018-02-01T14:28:42+0200')
>>> print(datetime_to_str(date))
01/02/2018
"""
try:
return date.strftime(pattern)
except ValueError as err:
raise ConversionException('Cannot convert {} to string with the format {}. '
'The following error occurred : {}'
.format(date, pattern, err))
def datetime_to_iso(date=now()):
"""Retrieves a date formatted to ISO format
Args:
date (datetime.datetime): The datetime object to convert. Defaults to now()
Returns:
str: The datetime object converted to string
Examples:
>>> date = str_to_datetime('2018-02-01T14:28:42+0200')
>>> print(datetime_to_iso(date))
2018-02-01T14:28:42+02:00
"""
return date.isoformat()
def parse_date(date_string, date_formats=None, languages=None, locales=None, region=None, settings=None):
"""Parse date and time from given date string.
@see dateparser.parse for more information
"""
def format_list_opt(opt):
return opt if opt is None or isinstance(opt, list) else [opt]
try:
res = dateparser.parse(
date_string,
date_formats=format_list_opt(date_formats),
languages=format_list_opt(languages),
locales=format_list_opt(locales),
region=region,
settings=settings)
if res is not None:
return res
raise ConversionException(
'Cannot convert {} to datetime'.format(date_string))
except ValueError as err:
raise ConversionException(
'Cannot convert {} to datetime. The following error occurs : {}'.format(date_string, err))
def str_to_datetime(value, pattern=DEFAULT_ISO_FORMAT):
"""Converts a string to a datetime from a pattern.
Args:
value (str): A string representing the date to convert
pattern (str): The format of the date to convert. Defaults to '%Y-%m-%dT%H:%M:%S%z' or '%Y-%m-%dT%H:%M:%S',
depending on python version.
See https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
Returns:
datetime.datetime: The converted datetime
Raises:
ConversionException: if conversion failed (bad pattern, ...)
Examples:
>>> print(str_to_datetime('20180201','%Y%m%d'))
2018-02-01 00:00:00
>>> print(str_to_datetime('2018-02-01T14:28:42+0200'))
2018-02-01 14:28:42+02:00
"""