Commit d3cf2aaa authored by BODERE's avatar BODERE
Browse files

Features/documentation

parent a2cea61a
......@@ -33,7 +33,6 @@ nosetests.xml
.project
.pydevproject
*.pyc
*.pyo
DS_Store
......@@ -44,34 +43,7 @@ VERSION.txt
migrations
*/migrations/*.py
src/datamodel/test/data/GW_L2P_ALT_ENVI_GDR_20110101_061209_20110101_070215_098_078.nc
src/datamodel/test/data/polarfront_2008-09-09T050025_2008-10-17T114909_HiWASE_2008_FORE.nc
src/datamodel/test/data/QS_S2B54289.20093261508
src/datamodel/test/data/WMO51101_20110101T0050_20110131T2350_Lat_24.32N_Lon_162.06W.nc
src/datamodel/test/data/20120109000000-OSISAF-L3C_GHRSST-SSTsubskin-SEVIRI_SST-ssteqc_meteosat09_20120109_000000-v02.0-fv01.0.nc
src/datamodel/test/data/20120323-ATS_NR_2P-UPA-L2P-ATS_NR__2PNPDK20120323_185109_000062203113_00042_52642_2614-v01.nc
src/20130404-NCDC-L4LRblend-GLOB-v01-fv02_0-AVHRR_OI.nc
src/20130516-IFR-L4_GHRSST-SSTfnd-ODYSSEA-GLOB_010-v2.0-fv1.0.nc
src/datamodel/fastregrid.c
src/dmi.nc
src/hrdds1.nc
src/hrdds2.nc
src/test.py
cerbere/20130404-NCDC-L4LRblend-GLOB-v01-fv02_0-AVHRR_OI.nc
cerbere/20130516-IFR-L4_GHRSST-SSTfnd-ODYSSEA-GLOB_010-v2.0-fv1.0.nc
cerbere/datamodel/fastregrid.c
cerbere/datamodel/test/data/20120109000000-OSISAF-L3C_GHRSST-SSTsubskin-SEVIRI_SST-ssteqc_meteosat09_20120109_000000-v02.0-fv01.0.nc
cerbere/datamodel/test/data/20120323-ATS_NR_2P-UPA-L2P-ATS_NR__2PNPDK20120323_185109_000062203113_00042_52642_2614-v01.nc
cerbere/datamodel/test/data/GW_L2P_ALT_ENVI_GDR_20110101_061209_20110101_070215_098_078.nc
cerbere/datamodel/test/data/QS_S2B54289.20093261508
cerbere/datamodel/test/data/WMO51101_20110101T0050_20110131T2350_Lat_24.32N_Lon_162.06W.nc
cerbere/datamodel/test/data/polarfront_2008-09-09T050025_2008-10-17T114909_HiWASE_2008_FORE.nc
cerbere/dmi.nc
cerbere/hrdds1.nc
cerbere/hrdds2.nc
tests/data/20120109000000-OSISAF-L3C_GHRSST-SSTsubskin-SEVIRI_SST-ssteqc_meteosat09_20120109_000000-v02.0-fv01.0.nc
tests/data/20120323-ATS_NR_2P-UPA-L2P-ATS_NR__2PNPDK20120323_185109_000062203113_00042_52642_2614-v01.nc
tests/data/GW_L2P_ALT_ENVI_GDR_20110101_061209_20110101_070215_098_078.nc
tests/data/QS_S2B54289.20093261508
tests/data/WMO51101_20110101T0050_20110131T2350_Lat_24.32N_Lon_162.06W.nc
tests/data/polarfront_2008-09-09T050025_2008-10-17T114909_HiWASE_2008_FORE.nc
/src/
/cerbere/**/*.nc
/tests/data/
/public/
# ---------------------------------------------------------------
# Global
# ---------------------------------------------------------------
# default image
image: continuumio/miniconda3:latest
# stages (main steps of pipeline)
stages:
# - Quality
# - Tests
# - Sonarqube
- Documentation
# - Deploy
# ---------------------------------------------------------------
# Jobs templates
# ---------------------------------------------------------------
.code-changes-template: &code-changes
only:
changes:
- ${CI_PROJECT_NAME}/**/*.py
- tests/**/*.py
- .gitlab-ci.yml
- pyproject.toml
.install-deps-conda-template-37: &install-conda-deps-37
before_script:
- conda create -n cerbere python=3.7
- source activate cerbere
- conda install -c conda-forge pygrib
- pip install poetry poetry-dynamic-versioning poetry2conda
- poetry config virtualenvs.in-project false
- poetry install -vv
tags: [cerbere-runner]
.install-deps-conda-template-38: &install-conda-deps-38
before_script:
- conda create -n cerbere python=3.8
- source activate cerbere
- conda install -c conda-forge pygrib
- pip install poetry poetry-dynamic-versioning poetry2conda
- poetry config virtualenvs.in-project false
- poetry install -vv
tags: [cerbere-runner]
.install-deps-conda-template-39: &install-conda-deps-39
before_script:
- conda create -n cerbere python=3.9
- source activate cerbere
- conda install -c conda-forge pygrib
- pip install poetry poetry-dynamic-versioning poetry2conda
- poetry config virtualenvs.in-project false
- poetry install -vv
tags: [cerbere-runner]
.quality-template: &quality
<<: *install-conda-deps-38
<<: *code-changes
stage: Quality
allow_failure: true
except:
- tags
.test-template-37: &test_37
<<: *install-conda-deps-37
<<: *code-changes
stage: Tests
script:
- poetry run pytest tests/
except:
- tags
.test-template-38: &test_38
<<: *install-conda-deps-38
<<: *code-changes
stage: Tests
script:
- poetry run pytest tests/
except:
- tags
# ---------------------------------------------------------------
# Quality jobs
# ---------------------------------------------------------------
#flake8:
# <<: *quality
# script:
# - poetry run flake8 --max-line-length=120 --docstring-convention google ${CI_PROJECT_NAME}
#
#pylint:
# <<: *quality
# script:
# #- poetry run pylint -j8 -E ${CI_PROJECT_NAME} tests
# - "poetry run pylint --exit-zero ${CI_PROJECT_NAME} tests -r n --max-line-length=120 --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' | tee pylint.txt"
# artifacts:
# expire_in: 1 week
# paths:
# - pylint.txt
# ---------------------------------------------------------------
# Test jobs
# ---------------------------------------------------------------
#python3.7:
# <<: *test_37
# image: continuumio/miniconda3
# allow_failure: true
# script:
# - poetry run pytest tests/
#
#python3.8:
# <<: *test_38
# image: continuumio/miniconda3
# allow_failure: true
# script:
# - poetry run pytest tests/
#
#python3.9:
# <<: *install-conda-deps-39
# <<: *code-changes
# stage: Tests
# image: continuumio/miniconda3
# coverage: '/^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+\%)/'
# allow_failure: true
# script:
# - poetry run pytest --cov=${CI_PROJECT_NAME} --cov-branch --cov-report=term tests/
# - poetry run pytest -ra -q --cov=${CI_PROJECT_NAME} --cov-branch --cov-report xml:coverage.xml tests/
# artifacts:
# expire_in: 1 week
# paths:
# - coverage.xml
# except:
# - tags
# ---------------------------------------------------------------
# SonarQube
# ---------------------------------------------------------------
#sonarqube:
# stage: Sonarqube
# tags: [cerbere-runner]
# image:
# name: sonarsource/sonar-scanner-cli:latest
# entrypoint: [""]
# allow_failure: true
# script:
# - sonar-scanner
# -Dsonar.projectKey=${CI_PROJECT_NAME}
# -Dsonar.language=py
# -Dsonar.host.url=http://visi-common-sonar:9000
# -Dsonar.login=e6f816eee72d3d5c03319ec74b468157b9164d12
# -Dsonar.sourceEncoding=UTF-8
# -Dsonar.python.coverage.reportPaths=coverage.xml
# -Dsonar.coverage.exclusions=**__init__**,tests/**
# -Dsonar.python.pylint.reportPath=pylint.txt
# <<: *code-changes
# except:
# - tags
# ---------------------------------------------------------------
# Documentation job
# ---------------------------------------------------------------
pages:
<<: *install-conda-deps-38
image: continuumio/miniconda3 # does not work with python 3.9
stage: Documentation
#allow_failure: true
script:
- poetry run mkdocs build -f docs/mkdocs.yml -v
- mv docs/site public
only:
refs:
- master
changes:
- docs/**/*
- .gitlab-ci.yml
artifacts:
paths:
- public
except:
- tags
## ---------------------------------------------------------------
## Release job
## ---------------------------------------------------------------
#nexus:
# <<: *install-conda-deps-38
# stage: Deploy
# script:
# - poetry build --format wheel
# - poetry config repositories.nexus-public-release https://nexus-test.ifremer.fr/repository/hosted-pypi-public-release/
# - poetry publish -r nexus-public-release -u nexus-ci -p w2bH2NjgFmQnzVk3
# only:
# - tags
......@@ -5,3 +5,70 @@ CERBERE
The IFREMER / CERSAT geospatial processing tool.
See online documentation at: http://cerbere.readthedocs.org
---------------
Development
---------------
Create conda env
=================
.. code-block:: shell
conda creaate -n cerbere -c conda-forge python=3.7 pygrib netcdf4 pyproj xarray
Install poetry
=================
.. code-block:: shell
pip install poetry poetry-dynamic-versioning poetry2conda
poetry --version
poetry config repositories.nexus-public-release https://nexus-test.ifremer.fr/repository/hosted-pypi-public-release/
retrieve and install project
=============================
.. code-block:: shell
git clone https://gitlab.ifremer.fr/cerbere/cerbere.git
poetry install -v --no-root
List dependencies
==================
.. code-block:: shell
poetry show --tree
---------------
build
---------------
wheel
==================
.. code-block:: shell
poetry build --format wheel
poetry publish -r nexus-public-release -u nexus-ci -p w2bH2NjgFmQnzVk3
conda
==================
.. code-block:: shell
mkdir -p dist/conda
conda build assets/conda -c conda-forge --output-folder dist/conda
documentation
==================
.. code-block:: shell
poetry run sphinx-build -b html docs public
# -----------------------------------------------------------
# Package metadata
# -----------------------------------------------------------
package:
name: cerbere
version: 0.1.0
about:
home: https://gitlab.ifremer.fr/cerbere/cerbere
license: GPLv3
license_file: LICENSE.md
# -----------------------------------------------------------
# Package build
# -----------------------------------------------------------
# extra conda channels
channel:
- conda-forge
# path to root package
source:
path: ../../
# command to install package (here from wheel)
build:
#script: pip install ./dist/cerbere-*.whl
script: "poetry install --no-dev"
requirements:
# to build package
build:
- python
- pip
- wheel
# to be used at runtime
run:
- python #
- pyyaml # pypi : PyYAML
- netcdf4 # pypi : netCDF4
- scipy # pypi : scipy
- shapely # pypi : shapely
- xarray # pypi : xarray
#test:
# imports:
# - cerbere
[[package]]
name = "alabaster"
version = "0.7.12"
description = "A configurable sidebar-enabled Sphinx theme"
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "astroid"
version = "2.7.2"
description = "An abstract syntax tree for Python with inference support."
category = "dev"
optional = false
python-versions = "~=3.6"
[package.dependencies]
lazy-object-proxy = ">=1.4.0"
typed-ast = {version = ">=1.4.0,<1.5", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""}
typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""}
wrapt = ">=1.11,<1.13"
[[package]]
name = "atomicwrites"
version = "1.4.0"
description = "Atomic file writes."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "attrs"
version = "21.2.0"
description = "Classes Without Boilerplate"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.extras]
dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"]
docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"]
tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"]
[[package]]
name = "babel"
version = "2.9.1"
description = "Internationalization utilities"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.dependencies]
pytz = ">=2015.7"
[[package]]
name = "certifi"
version = "2021.5.30"
description = "Python package for providing Mozilla's CA Bundle."
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "cftime"
version = "1.5.0"
description = "Time-handling functionality from netcdf4-python"
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
numpy = "*"
[[package]]
name = "charset-normalizer"
version = "2.0.4"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "dev"
optional = false
python-versions = ">=3.5.0"
[package.extras]
unicode_backport = ["unicodedata2"]
[[package]]
name = "colorama"
version = "0.4.4"
description = "Cross-platform colored terminal text."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "coverage"
version = "5.5"
description = "Code coverage measurement for Python"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
[package.extras]
toml = ["toml"]
[[package]]
name = "docutils"
version = "0.16"
description = "Docutils -- Python Documentation Utilities"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "flake8"
version = "3.9.2"
description = "the modular source code checker: pep8 pyflakes and co"
category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
[package.dependencies]
importlib-metadata = {version = "*", markers = "python_version < \"3.8\""}
mccabe = ">=0.6.0,<0.7.0"
pycodestyle = ">=2.7.0,<2.8.0"
pyflakes = ">=2.3.0,<2.4.0"
[[package]]
name = "flake8-docstrings"
version = "1.6.0"
description = "Extension for flake8 which uses pydocstyle to check docstrings"
category = "dev"
optional = false
python-versions = "*"
[package.dependencies]
flake8 = ">=3"
pydocstyle = ">=2.1"
[[package]]
name = "idna"
version = "3.2"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "dev"
optional = false
python-versions = ">=3.5"
[[package]]
name = "imagesize"
version = "1.2.0"
description = "Getting image size from png/jpeg/jpeg2000/gif file"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "importlib-metadata"
version = "4.7.1"
description = "Read metadata from Python packages"
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""}
zipp = ">=0.5"
[package.extras]
docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
perf = ["ipython"]
testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
[[package]]
name = "iniconfig"
version = "1.1.1"
description = "iniconfig: brain-dead simple config-ini parsing"
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "isort"
version = "5.9.3"
description = "A Python utility / library to sort Python imports."
category = "dev"
optional = false
python-versions = ">=3.6.1,<4.0"
[package.extras]
pipfile_deprecated_finder = ["pipreqs", "requirementslib"]
requirements_deprecated_finder = ["pipreqs", "pip-api"]
colors = ["colorama (>=0.4.3,<0.5.0)"]
plugins = ["setuptools"]
[[package]]
name = "jinja2"
version = "3.0.1"
description = "A very fast and expressive template engine."
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
MarkupSafe = ">=2.0"
[package.extras]
i18n = ["Babel (>=2.7)"]
[[package]]
name = "lazy-object-proxy"
version = "1.6.0"
description = "A fast and thorough lazy object proxy."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
[[package]]
name = "markupsafe"
version = "2.0.1"
description = "Safely add untrusted strings to HTML/XML markup."
category = "dev"
optional = false
python-versions = ">=3.6"
[[package]]
name = "mccabe"
version = "0.6.1"
description = "McCabe checker, plugin for flake8"
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "netcdf4"
version = "1.5.7"
description = "Provides an object-oriented python interface to the netCDF version 4 library."
category = "main"
optional = false
python-versions = "*"
[package.dependencies]
cftime = "*"
numpy = ">=1.9"
[[package]]
name = "numpy"
version = "1.21.1"
description = "NumPy is the fundamental package for array computing with Python."
category = "main"
optional = false
python-versions = ">=3.7"
[[package]]
name = "packaging"
version = "21.0"
description = "Core utilities for Python packages"
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
pyparsing = ">=2.0.2"
[[package]]
name = "pandas"
version = "1.1.5"
description = "Powerful data structures for data analysis, time series, and statistics"
category = "main"
optional = false
python-versions = ">=3.6.1"
[package.dependencies]
numpy = ">=1.15.4"
python-dateutil = ">=2.7.3"
pytz = ">=2017.2"
[package.extras]
test = ["pytest (>=4.0.2)", "pytest-xdist", "hypothesis (>=3.58)"]
[[package]]
name = "platformdirs"
version = "2.2.0"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
category = "dev"
optional = false
python-versions = ">=3.6"
[package.extras]
docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"]
test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"]
[[package]]
name = "pluggy"
version = "0.13.1"
description = "plugin and hook calling mechanisms for python"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.dependencies]
importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
[package.extras]