Commit 7f6dee7a authored by BODERE's avatar BODERE
Browse files

feat: poetry + gitlab

parent c5df2079
Pipeline #11396 failed with stages
...@@ -6,3 +6,5 @@ ...@@ -6,3 +6,5 @@
.project .project
dist dist
eo_dataflow_manager.egg-info eo_dataflow_manager.egg-info
/unittests/_build
\ No newline at end of file
# ---------------------------------------------------------------
# Global
# ---------------------------------------------------------------
# default image
image: python:latest
# stages (main steps of pipeline)
stages:
- Quality
- Tests
- Sonarqube
# - Documentation
- Deploy
# ---------------------------------------------------------------
# Jobs templates
# ---------------------------------------------------------------
.code-changes-template: &code-changes
only:
changes:
- ${CI_PROJECT_NAME}/**/*.py
- tests/**/*.py
- .gitlab-ci.yml
- pyproject.tomlx
.install-deps-template: &install-deps
before_script:
- pip install poetry poetry-dynamic-versioning poetry2conda
- poetry config virtualenvs.in-project false
- poetry install -vv
tags: [ cersat-runner ]
.quality-template: &quality
<<: *install-deps
<<: *code-changes
stage: Quality
allow_failure: true
except:
- tags
.test-template: &test
<<: *install-deps
<<: *code-changes
stage: Tests
allow_failure: true
script:
- poetry run pytest tests/
except:
- tags
# ---------------------------------------------------------------
# Quality jobs
# ---------------------------------------------------------------
flake8:
<<: *quality
script:
- "poetry run flake8 --max-line-length=120 ${CI_PROJECT_NAME}"
pylint:
<<: *quality
script:
#- poetry run pylint -j8 -E ${CI_PROJECT_NAME} tests
- "poetry run pylint --exit-zero ${CI_PROJECT_NAME} tests -r n --max-line-length=120 --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' | tee pylint.txt"
artifacts:
expire_in: 1 week
paths:
- pylint.txt
# ---------------------------------------------------------------
# Test jobs
# ---------------------------------------------------------------
python3.7:
<<: *test
image: python:3.7
script:
- poetry run pytest tests/
python3.8:
<<: *test
image: python:3.8
script:
- poetry run pytest tests/
python3.9:
<<: *install-deps
<<: *code-changes
stage: Tests
image: python:3.9
coverage: '/^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+\%)/'
script:
- poetry run pytest --cov=${CI_PROJECT_NAME} --cov-branch --cov-report=term tests/
- poetry run pytest -ra -q --cov=${CI_PROJECT_NAME} --cov-branch --cov-report xml:coverage.xml tests/
artifacts:
expire_in: 1 week
paths:
- coverage.xml
except:
- tags
# ---------------------------------------------------------------
# SonarQube
# ---------------------------------------------------------------
sonarqube:
stage: Sonarqube
tags: [ cersat-runner ]
image:
name: sonarsource/sonar-scanner-cli:latest
entrypoint: [ "" ]
allow_failure: true
script:
- sonar-scanner
-Dsonar.projectKey=${CI_PROJECT_NAME}
-Dsonar.language=py
-Dsonar.host.url=http://visi-common-sonar:9000
-Dsonar.login=e6f816eee72d3d5c03319ec74b468157b9164d12
-Dsonar.sourceEncoding=UTF-8
-Dsonar.python.coverage.reportPaths=coverage.xml
-Dsonar.coverage.exclusions=**__init__**,tests/**
-Dsonar.python.pylint.reportPath=pylint.txt
<<: *code-changes
except:
- tags
# ---------------------------------------------------------------
# Documentation job
# ---------------------------------------------------------------
#pages:
# <<: *install-deps
# image: python:3.8 # does not work with python 3.9
# stage: Documentation
# script:
# - poetry run mkdocs build -f docs/mkdocs.yml -v
# - mv docs/site public
# only:
# refs:
# - master
# changes:
# - docs/**/*
# - .gitlab-ci.yml
# artifacts:
# paths:
# - public
# except:
# - tags
# ---------------------------------------------------------------
# Release job
# ---------------------------------------------------------------
nexus:
<<: *install-deps
stage: Deploy
script:
- poetry build --format wheel
- poetry config repositories.nexus-public-release https://nexus-tests.ifremer.fr/repository/hosted-pypi-public-release/
- poetry publish -r nexus-public-release -u nexus-ci -p w2bH2NjgFmQnzVk3
only:
- tags
BODERE <erwan.bodere@ifremer.fr>
Erwan BODERE <erwan.bodere@ifremer.fr>
F. PAUL <fpaul@ifremer.fr>
Fabien CADORET <fcadoret@br152-128.ifremer.fr>
Frederic PAUL <fpaul@hancock.ifremer.fr>
MAISSIAT <pierre.maissiat@partenaire-exterieur.ifremer.fr>
Pierre MAISSIAT <pmaissia@ananda.ifremer.fr>
Sylvain Herlédan <sylvain.herledan@shadowz.sytes.net>
Thibaut CHARLES <thibaut.charles@ifremer.fr>
bcauseur <bcauseur@sii.fr>
jfpiolle <jfpiolle@ifremer.fr>
medspi <medspi@br156-133.ifremer.fr>
pifgold <pifgold@free.fr>
#!/usr/bin/env bash
set -e
## Hist :
##
########################################################################################
###
### USAGE START DOWNMOADER SCRIPT
###
########################################################################################
function usage
{
echo "Usage: `basename $0` -a|--appdata_dir <applicationdata_path> [<options>]"
echo "options :"
echo " -h, --help"
echo " -a, --appdata_dir"
echo " -f, --force"
echo " -u, --update"
}
########################################################################################
###
### DOWNLOADER CONF INSTALLING SCRIPT AND BUILD CONFIGURATION FILE
###
########################################################################################
function init()
{
set OPTION=
if [[ ${FORCE} == "YES" || ${UPDATE} == "YES" ]]; then
if [[ ${FORCE} == "YES" ]]; then
echo " => Reinstalling the ${DOWNLOADER_APPDATA} application data"
rm -rf ${DOWNLOADER_APPDATA}/*
else
echo " => Update of the ${DOWNLOADER_APPDATA} application data"
OPTION=-n
fi
else
echo " => Initializing the ${DOWNLOADER_APPDATA} application data"
fi
DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [[ $CONDA_PREFIX != "" ]]; then
cp -r $CONDA_PREFIX/usr/share/eo-dataflow-manager/* ${DOWNLOADER_APPDATA} ${OPTION}
else
cp -r ${DOWNLOADERPACKAGENAME}/conf/* ${DOWNLOADER_APPDATA} ${OPTION}
fi
echo " => Building the configuration file ${DOWNLOADER_APPD}"
echo "#!/usr/bin/env bash" > ${DOWNLOADER_APPD}
echo "#" >> ${DOWNLOADER_APPD}
echo "# Downloader version ${DOWNLOADER_VERSION}" >> ${DOWNLOADER_APPD}
echo "# Deployment date:" `date` >> ${DOWNLOADER_APPD}
echo "#" >> ${DOWNLOADER_APPD}
echo "export DOWNLOADER_APPD=${DOWNLOADER_APPD}" >> ${DOWNLOADER_APPD}
echo "export PYTHONPATH=${PYTHONPATH}" >> ${DOWNLOADER_APPD}
echo "#" >> ${DOWNLOADER_APPD}
echo "export DOWNLOADER_CONF_VERSION=${DOWNLOADER_VERSION}" >> ${DOWNLOADER_APPD}
echo "#" >> ${DOWNLOADER_APPD}
}
########################################################################################
###
### PARSER THE COMMAND LINE
###
########################################################################################
function parser()
{
POSITIONAL=()
COMMANDE_LINE=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-a|--appdata_dir)
export DOWNLOADER_APPDATA="$2"
shift # past argument
shift # past value
;;
-h|--help)
HELP="YES"
shift # past argument
;;
-f|--force)
FORCE="YES"
shift # past argument
;;
-u|--update)
UPDATE="YES"
shift # past argument
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
}
############################################################################################
###
### MAIN INIT_CONF SCRIPT
###
############################################################################################
if [[ $(id -u) -eq 0 ]]; then
echo "[ERROR] This script should not be started as root !" 2>&1
exit 1
fi
DOWNLOADER_ROOT=
HELP="NO"
FORCE="NO"
UPDATE="NO"
parser $*
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show eo-dataflow-manager | grep Version | cut -c 10-`
echo "Downloader version = ${DOWNLOADER_VERSION}"
echo "Downloader appdata = ${DOWNLOADER_APPDATA}"
#echo "POSITIONAL = ${POSITIONAL}"
#echo "COMMANDE_LINE = ${COMMANDE_LINE[@]}"
#Internal constants
SCRIPT_DIR=$(dirname $(readlink -f "$0"))
export PYTHONPATH=${SCRIPT_DIR}
export DOWNLOADER_APPD=${DOWNLOADER_APPDATA}/eo-dataflow-manager.appd
export DOWNLOADER_PATTERN="[p]ython.*eo_dataflow_manager.py --workspace_dir * --appdata_dir ${DOWNLOADER_APPDATA}"
#External constants
if [[ $HELP == "YES" ]]; then
usage
exit 0
fi
if [[ -z ${DOWNLOADER_APPDATA} ]]; then
echo "Error : DOWNLOADER_APPDATA not set"
usage
exit 1
fi
if [[ ! -z "${POSITIONAL[@]}" ]]; then
echo "Error: unknown option(s): " ${POSITIONAL[@]}
usage
exit 1
fi
if [[ ${FORCE} == "YES" && ${UPDATE} == "YES" ]]; then
echo "Error: the "-f" and "-u" options are incompatible !"
exit 1
fi
if [[ ${FORCE} == "YES" && ${UPDATE} == "YES" ]]; then
echo "Error: the "-f" and "-u" options are incompatible !"
exit 1
fi
################################################################
### VERIFY EXISTING WORKSPACE ROOT DIRECTORY
################################################################
if [[ ! -d ${DOWNLOADER_APPDATA} ]]; then
echo "Error: application data directory (${DOWNLOADER_APPDATA}) not exists, first create the application data directory"
exit 1
fi
################################################################
### VERIFY DOWNLOADER INSTALLATION
################################################################
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [ -z ${DOWNLOADERPACKAGENAME} ]; then
echo "Error: downloader package is not installed, check the path to the active Python."
exit 1
fi
if [[ ${FORCE} == "NO" && ${UPDATE} == "NO" ]]; then
if [ -f ${DOWNLOADER_APPD} ]; then
echo "Error: configuration file (${DOWNLOADER_APPD}) already exists, use the '--force' option to force the reinstallation or the --update option to update the existing application data"
exit 1
fi
if (( $(ls -l ${DOWNLOADER_APPDATA}/ | wc -l) > 1 )); then
echo "Error: application data directory is not empty, use the '--force' option to force the reinstallation or the --update option to update the existing application data"
exit 1
fi
fi
################################################################
### VERIFY NO DOWNLOADER RUNNING WITH APPLICATION DATA DIRECTORY
################################################################
if [[ $(ps -ef | grep "$DOWNLOADER_PATTERN" | wc -l) -eq 1 ]]; then
echo "Error : a downloader is already running with this application data directory, stop it first"
exit 1
fi
echo "Environment OK ..."
################################################################
### INSTALLING DOWNLOADER CONFIGURATION
################################################################
init
echo "*** The downloader configuration is initialized ***"
#!/usr/bin/env bash
## Hist :
##
########################################################################################
###
### USAGE START DOWNMOADER SCRIPT
###
########################################################################################
function usage
{
echo "Usage: `basename $0` -w|--workspace_dir <workspace_path> [<options>]"
echo "options :"
echo " -h, --help"
echo " -f, --force"
echo " -u, --update"
}
########################################################################################
###
### DOWNLOADER WORKSPACE AND CONF INSTALLING SCRIPT
###
########################################################################################
function init()
{
set OPTION=
if [[ ${FORCE} == "YES" || ${UPDATE} == "YES" ]]; then
if [[ ${FORCE} == "YES" ]]; then
echo " => Reinstalling the ${DOWNLOADER_WORKSPACE} workspace "
rm -rf ${DOWNLOADER_WORKSPACE}/*
else
echo " => Update of the ${DOWNLOADER_WORKSPACE} workspace"
OPTION=-p
fi
else
echo " => Creating the ${DOWNLOADER_WORKSPACE} workspace"
fi
mkdir ${DOWNLOADER_WORKSPACE}/cache ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/cache/jobfile ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/cache/providerfile ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/cache/synchrofile ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/cache/plugins ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/downloads ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/log ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/log/messages_archive ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/log/messages_emergency ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/log/runner ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/spools ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/spools/message ${OPTION}
mkdir ${DOWNLOADER_WORKSPACE}/spools/monitoring ${OPTION}
echo " => Building the workspace file ${DOWNLOADER_WRKS}"
echo "#!/usr/bin/env bash" > ${DOWNLOADER_WRKS_FILE}
echo "#" >> ${DOWNLOADER_WRKS_FILE}
echo "# Downloader version ${DOWNLOADER_VERSION}" >> ${DOWNLOADER_WRKS_FILE}
echo "# Deployment date:" `date` >> ${DOWNLOADER_WRKS_FILE}
echo "#" >> ${DOWNLOADER_WRKS_FILE}
echo "export DOWNLOADER_WRKS_VERSION=${DOWNLOADER_VERSION}" >> ${DOWNLOADER_WRKS_FILE}
echo "#" >> ${DOWNLOADER_WRKS_FILE}
echo "export DOWNLOADER_PATTERN='${DOWNLOADER_PATTERN}'" >> ${DOWNLOADER_WRKS_FILE}
echo "export DOWNLOADER_LOCK=${DOWNLOADER_LOCK}" >> ${DOWNLOADER_WRKS_FILE}
echo "#" >> ${DOWNLOADER_WRKS_FILE}
}
########################################################################################
###
### PARSER THE COMMAND LINE
###
########################################################################################
function parser()
{
POSITIONAL=()
COMMANDE_LINE=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-w|--workspace_dir)
export DOWNLOADER_WORKSPACE=`echo "$2"|sed -e "s://*:/:g;s:/$::"`
shift # past argument
shift # past value
;;
-h|--help)
HELP="YES"
shift # past argument
;;
-f|--force)
FORCE="YES"
shift # past argument
;;
-u|--update)
UPDATE="YES"
shift # past argument
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
}
############################################################################################
###
### MAIN INIT_WORKSPACE SCRIPT
###
############################################################################################
if [[ $(id -u) -eq 0 ]]; then
echo "[ERROR] This script should not be started as root !" 2>&1
exit 1
fi
DOWNLOADER_WORKSPACE=
HELP="NO"
FORCE="NO"
UPDATE="NO"
parser $*
export DOWNLOADER_VERSION=`pip --disable-pip-version-check show eo-dataflow-manager | grep Version | cut -c 10-`
echo "Downloader version = ${DOWNLOADER_VERSION}"
echo "Downloader workspace = ${DOWNLOADER_WORKSPACE}"
#echo "POSITIONAL = ${POSITIONAL}"
#echo "COMMANDE_LINE = ${COMMANDE_LINE[@]}"
#Internal constants
SCRIPT_DIR=$(dirname $(readlink -f "$0"))
export PYTHONPATH=${SCRIPT_DIR}
export DOWNLOADER_WRKS_FILE=${DOWNLOADER_WORKSPACE}/eo-dataflow-manager.wrks
export DOWNLOADER_LOCK="${DOWNLOADER_WORKSPACE}/log/scheduler.log.lock"
export DOWNLOADER_PATTERN="[p]ython.*eo_dataflow_manager.py --workspace_dir ${DOWNLOADER_WORKSPACE}"
if [[ $HELP == "YES" ]]; then
usage
exit 0
fi
if [[ -z ${DOWNLOADER_WORKSPACE} ]]; then
echo "Error : DOWNLOADER_WORKSPACE not set"
usage
exit 1
fi
if [[ ! -z "${POSITIONAL[@]}" ]]; then
echo "Error: unknown option(s): " ${POSITIONAL[@]}
usage
exit 1
fi
if [[ ${FORCE} == "YES" && ${UPDATE} == "YES" ]]; then
echo "Error: the "-f" and "-u" options are incompatible !"
exit 1
fi
################################################################
### VERIFY EXISTING WORKSPACE ROOT DIRECTORY
################################################################
if [[ ! -d ${DOWNLOADER_WORKSPACE} ]]; then
echo "Error: workspace directory (${DOWNLOADER_WORKSPACE}) not exists, first create the directory of the workspace"
exit 1
fi
################################################################
### VERIFY DOWNLOADER INSTALLATION
################################################################
export DOWNLOADERPACKAGENAME=`python -c 'import sys; import os; t=__import__("eo-dataflow-manager"); print(os.path.dirname(t.__file__))' 2> /dev/null`
if [ -z ${DOWNLOADERPACKAGENAME} ]; then
echo "Error: downloader package is not installed, check the path to the active Python."
exit 1
fi
if [[ ${FORCE} == "NO" && ${UPDATE} == "NO" ]]; then
if [ -f ${DOWNLOADER_WRKS_FILE} ]; then
echo "Error: configuration file (${DOWNLOADER_WRKS_FILE}) already exists, use the '--force' option to force the reinstallation or the --update option to update the existing installation"
exit 1
fi
listfiles=($(ls ${DOWNLOADER_WORKSPACE}/*))
if [[ ${#listfiles[@]} -gt 1 ]]; then
echo "Error: root workspace directory is not empty, use the '--force' option to force the reinstallation or the --update option to update the existing installation"
exit 1
fi
fi
################################################################
### VERIFY NO DOWNLOADER RUNNING
################################################################
if [ -f ${DOWNLOADER_LOCK} ]; then
if [[ $(ps `awk -F$HOSTNAME '{print $(NF)}' $DOWNLOADER_LOCK` | grep "$DOWNLOADER_PATTERN" | wc -l) -eq 1 ]]; then
echo "Error : a downloader is already running in this workspace, stop it first"
exit 1
fi
fi
################################################################
### INSTALLING WORKSPACE
################################################################
echo "Environment OK ..."
init
echo "*** The downloader workspace is initialized ***"
downloadFile :
fabrique un DownloadedFile après téléchargement de la donnée
AJOUT INFOS SUR LE DOWNLOAD FILE
- data_for_day