Commit 3d289e74 authored by SOUVY's avatar SOUVY
Browse files

cersat/general#142 cerbere poetry packaging, doc with sphinx

parent d2b6e027
# ---------------------------------------------------------------
# Global
# ---------------------------------------------------------------
# default image
image: python:latest
# stages (main steps of pipeline)
stages:
- Quality
- Tests
- Sonarqube
#- Documentation
- Deploy
# ---------------------------------------------------------------
# Jobs templates
# ---------------------------------------------------------------
.code-changes-template: &code-changes
only:
changes:
- ${CI_PROJECT_NAME}/**/*.py
- tests/**/*.py
- .gitlab-ci.yml
- pyproject.toml
.install-deps-template: &install-deps
before_script:
- pip install poetry poetry-dynamic-versioning poetry2conda
- poetry config virtualenvs.in-project false
- poetry install -vv
tags: [cerbere]
.quality-template: &quality
<<: *install-deps
<<: *code-changes
stage: Quality
allow_failure: true
except:
- tags
.test-template: &test
<<: *install-deps
<<: *code-changes
stage: Tests
script:
- poetry run pytest tests/
except:
- tags
# ---------------------------------------------------------------
# Quality jobs
# ---------------------------------------------------------------
flake8:
<<: *quality
script:
- poetry run flake8 --max-line-length=120 --docstring-convention google ${CI_PROJECT_NAME}
pylint:
<<: *quality
script:
#- poetry run pylint -j8 -E ${CI_PROJECT_NAME} tests
- "poetry run pylint --exit-zero ${CI_PROJECT_NAME} tests -r n --max-line-length=120 --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' | tee pylint.txt"
artifacts:
expire_in: 1 week
paths:
- pylint.txt
# ---------------------------------------------------------------
# Test jobs
# ---------------------------------------------------------------
python3.7:
<<: *test
image: python:3.7
script:
- poetry run pytest tests/
python3.8:
<<: *test
image: python:3.8
script:
- poetry run pytest tests/
python3.9:
<<: *install-deps
<<: *code-changes
stage: Tests
image: python:3.9
coverage: '/^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+\%)/'
script:
- poetry run pytest --cov=${CI_PROJECT_NAME} --cov-branch --cov-report=term tests/
- poetry run pytest -ra -q --cov=${CI_PROJECT_NAME} --cov-branch --cov-report xml:coverage.xml tests/
artifacts:
expire_in: 1 week
paths:
- coverage.xml
except:
- tags
# ---------------------------------------------------------------
# SonarQube
# ---------------------------------------------------------------
sonarqube:
stage: Sonarqube
tags: [cerbere]
image:
name: sonarsource/sonar-scanner-cli:latest
entrypoint: [""]
allow_failure: true
script:
- sonar-scanner
-Dsonar.projectKey=${CI_PROJECT_NAME}
-Dsonar.language=py
-Dsonar.host.url=http://visi-common-sonar:9000
-Dsonar.login=e6f816eee72d3d5c03319ec74b468157b9164d12
-Dsonar.sourceEncoding=UTF-8
-Dsonar.python.coverage.reportPaths=coverage.xml
-Dsonar.coverage.exclusions=**__init__**,tests/**
-Dsonar.python.pylint.reportPath=pylint.txt
<<: *code-changes
except:
- tags
# ---------------------------------------------------------------
# Documentation job
# ---------------------------------------------------------------
#pages:
# <<: *install-deps
# image: python:3.8 # does not work with python 3.9
# stage: Documentation
# script:
# - poetry run mkdocs build -f docs/mkdocs.yml -v
# - mv docs/site public
# only:
# refs:
# - master
# changes:
# - docs/**/*
# - .gitlab-ci.yml
# artifacts:
# paths:
# - public
# except:
# - tags
# ---------------------------------------------------------------
# Release job
# ---------------------------------------------------------------
nexus:
<<: *install-deps
stage: Deploy
script:
- poetry build --format wheel
- poetry config repositories.nexus-public-release https://nexus-test.ifremer.fr/repository/hosted-pypi-public-release/
- poetry publish -r nexus-public-release -u nexus-ci -p w2bH2NjgFmQnzVk3
only:
- tags
Jean-François Piollé <jean.francois.piolle@ifremer.fr>
David Poulter <david.poulter@pelamis.co.uk>
Antoine Grouazel <antoine.grouazel@ifremer.fr>
Gilles Guitton <gilles.guitton@oceandatalab.com>
Sylvain Herlédan <sylvain.herledan@oceandatalab.com>
====================
Cerbere installation
====================
Requirements
============
Your Python package manager should handle most of the dependencies by himself, but there are some gotchas:
- Numpy is required by the setup.py scripts of other modules, so it must be installed beforehand (see https://github.com/pypa/pip/issues/25)::
pip install numpy==1.7.1
- The versioning convention of the pyhdf package changed from major.minor-revision to major.minor.revision. This change prevents package managers to handle pyhdf correctly, you have to install it manually
.. sourcecode :: bash
cd /tmp
wget "http://downloads.sourceforge.net/project/pysclint/pyhdf/0.8.3/pyhdf-0.8.3.tar.gz"
tar xvzf pyhdf-0.8.3.tar.gz
cd pyhdf-0.8.3
# Change the following variables depending on your environment
export INCLUDE_DIRS=/usr/include/hdf
export LIBRARY_DIRS=/usr/lib
export NOSZIP=1
python setup.py install
.. note::
If the installation fails, check that the directory you specified in LIBRARY_DIRS contains libmfhdf.so and libdf.so.
They may have been renamed, preventing the linker to find them.
For example, on Ubuntu 12.04, you can find these libraries as "libmfhdfalt.so" and "libdfalt.so".
To fix this, edit setup.py and set the correct names in the "libraries" variable (line 88).
On Ubuntu 12.04, you can use the install.sh script provided in the cerbere package to install dependencies.
Cerbere
=======
Just use your package manager to perform installation::
pip install .
This diff is collapsed.
This diff is collapsed.
include *.txt
include *.rst
include docs/*.rst
include cerbere/share/default_saving_profile.yaml
include cerbere/share/ghrsst_saving_profile.yaml
The IFREMER / CERSAT geospatial processing tool.
See online documentation at: http://cerbere.readthedocs.org
## Development
### Install poetry
```bash
pip install poetry poetry-dynamic-versioning poetry2conda
poetry --version
poetry config repositories.nexus-public-release https://nexus-test.ifremer.fr/repository/hosted-pypi-public-release/
```
### retrieve and install project
```bash
git clone https://gitlab.ifremer.fr/ifremer-commons/python/ifr_lib.git
git clone https://gitlab.ifremer.fr/cerbere/cerbere.git
poetry install -v --no-root
```
### Install pygrib with conda before installing cerbere
```bash
conda install -c conda-forge pygrib
```
### List dependencies
```bash
poetry show --tree
```
# build and publish wheel
```bash
poetry build --format wheel
poetry publish -r nexus-public-release -u nexus-ci -p w2bH2NjgFmQnzVk3
```
=======
CERBERE
=======
The IFREMER / CERSAT geospatial processing tool.
See online documentation at: http://cerbere.readthedocs.org
[build-system]
requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "cerbere"
version = "0.0.0"
description = "A python API to manipulate spatial and temporal observations, compatible with many existing formats, normalizing and typing the representation of these observations."
license = "AOSI Approved :: GNU General Public License v3 (GPLv3)"
authors = ["Jean-François Piollé (Ifremer) <jean.francois.piolle@ifremer.fr>"]
maintainers = ["Jean-François Piollé (Ifremer) <jean.francois.piolle@ifremer.fr>"]
readme = "README.md"
classifiers = [
"Development Status :: 4 - Beta",
"Environment :: Console",
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8"
]
keywords = ["ocean", "observation", "data", "analysis"]
exclude = ["tests"]
[tool.poetry-dynamic-versioning]
enable = true
vcs = "git"
style = "semver"
pattern = "^v?(?P<base>\\d+\\.\\d+\\.\\d+)(-?((?P<stage>[a-zA-Z]+)\\.?(?P<revision>\\d+)?))?$"
[[tool.poetry.source]]
name = "nexus-isi"
url = "https://nexus-test.ifremer.fr/repository/pypi-public-release/simple"
[tool.poetry.dependencies]
python = "^3.7"
numpy = "^1.19.1"
pandas = "^1.1.1"
xarray = "^0.16"
shapely = "^1.2.18"
python-dateutil = "^2.1"
cftime = "^1.2.1"
netCDF4 = "^1.5.4"
scipy = "^1.5.2"
pyyaml = "^5.3.1"
pyproj = "^2.6.1"
pygrib = "^2.0.5"
[tool.poetry.dev-dependencies]
# static analysis
flake8 = "^3.8"
flake8-docstrings = "^1.5"
pylint = "^2.6"
# unit tests
pytest = "^6"
pytest-cov = "^2"
testfixtures = "^6"
# documentation
sphinx_autodoc_typehints = "^1.11.0"
[tool.poetry.plugins."cerbere.plugins"]
Dataset = "cerbere.dataset.dataset:Dataset"
NCDataset = "cerbere.dataset.ncdataset:NCDataset"
GHRSSTNCDataset = "cerbere.dataset.ghrsstncdataset:GHRSSTNCDataset"
[tool.poetry.plugins."cerbere.features"]
Grid = "cerbere.feature.grid:Grid"
CylindricalGrid = "cerbere.feature.grid:CylindricalGrid"
Swath = "cerbere.feature.swath:Swath"
Image = "cerbere.feature.image:Image"
Trajectory = "cerbere.feature.trajectory:Trajectory"
GridTimeSeries = "cerbere.feature.gridtimeseries:GridTimeSeries"
CylindricalGridTimeSeries = "cerbere.feature.gridtimeseries:CylindricalGridTimeSerie"
Profile = "cerbere.feature.profile:Profile"
IMDCollection = "cerbere.feature.imdcollection:IMDCollection"
OMDCollection = "cerbere.feature.omdcollection:OMDCollection"
TrajectoryProfile = "cerbere.feature.trajectoryprofile:TrajectoryProfile"
TimeSeriesProfile = "cerbere.feature.timeseriesprofile:TimeSeriesProfile"
UniZTimeSeriesProfile = "cerbere.feature.timeseriesprofile:UniZTimeSeriesProfile"
# Use official repository
-i https://pypi.python.org/simple/
-e .
numpy
pandas
xarray>=0.15
shapely>=1.2.18
python-dateutil>=2.1
cftime
sphinx_autodoc_typehints
# encoding: utf-8
"""
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from setuptools import setup
__package_name__ = 'cerbere'
__version__ = '2.0.0'
with open("CONTRIBUTORS.txt", "r") as f:
authors_email = ', '.join(f.readlines())
setup(
name=__package_name__,
version=__version__,
author="Cerbere Collaborators: %s" % authors_email,
author_email="jean.francois.piolle@ifremer.fr",
description=(
"A python API to read and normalize earth science observations"
),
license="LICENSE.txt",
keywords="ocean observation data analysis",
url="https://gitlab.ifremer.fr/cerbere/cerbere.git",
use_scm_version=True,
setup_requires=['setuptools_scm'],
packages=[
'cerbere',
'cerbere.feature',
'cerbere.dataset',
'tests',
'cerbere.share',
],
include_package_data=True,
long_description=(
"a python API to manipulate spatial and temporal observations, "
"compatible with many existing formats, normalizing and typing the "
"representation of these observations."
),
install_requires=[
'netCDF4',
'xarray>=0.15',
'cftime',
'Shapely>=1.2.18',
'scipy',
'pyyaml'
],
entry_points={
'cerbere.plugins': [
'Dataset = cerbere.dataset.dataset:Dataset',
'NCDataset = cerbere.dataset.ncdataset:NCDataset',
'GHRSSTNCDataset = cerbere.dataset.ghrsstncdataset:GHRSSTNCDataset',
],
'cerbere.features': [
'Grid = cerbere.feature.grid:Grid',
'CylindricalGrid = cerbere.feature.grid:CylindricalGrid',
'Swath = cerbere.feature.swath:Swath',
'Image = cerbere.feature.image:Image',
'Trajectory = cerbere.feature.trajectory:Trajectory',
'GridTimeSeries = cerbere.feature.gridtimeseries:GridTimeSeries',
'CylindricalGridTimeSeries = cerbere.feature.gridtimeseries:CylindricalGridTimeSeries',
'Profile = cerbere.feature.profile:Profile',
'IMDCollection = cerbere.feature.imdcollection:IMDCollection',
'OMDCollection = cerbere.feature.omdcollection:OMDCollection',
'TrajectoryProfile = cerbere.feature.trajectoryprofile:TrajectoryProfile',
'TimeSeriesProfile = cerbere.feature.timeseriesprofile:TimeSeriesProfile',
'UniZTimeSeriesProfile = cerbere.feature.timeseriesprofile:UniZTimeSeriesProfile',
]
},
zip_safe=False,
classifiers=[
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'Topic :: Scientific/Engineering',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
],
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment