Commit 14079e34 authored by MAISSIAT's avatar MAISSIAT Committed by BODERE
Browse files

Hotfix 2.0.5

parent 983572e1
#
#
#
#------------------------------------------------------------------
FROM python:3.7-slim-stretch
SHELL ["/bin/bash", "--login", "-c"]
RUN apt -y update && \
apt -y install \
apache2 apache2-dev apache2-utils \
wget ca-certificates \
make gcc musl-dev libxslt-dev \
git
#------------------------------------------------------------------
RUN cd /home && \
git clone https://gitlab.ifremer.fr/downloader/downloader_ui.git && \
rm -rf /var/cache/apk/*
ENV PWD=/home/downloader_ui
WORKDIR ${PWD}
RUN /usr/local/bin/pip3 install -r requirements.txt \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/pypi \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-public-snapshot/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-private-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-private-snapshot/simple && \
/usr/local/bin/pip3 install --no-binary "mod_wsgi" mod_wsgi uwsgi && \
/usr/local/bin/pip3 install .
ENV USER_NAME=www-data GROUP_NAME=www-data
ENV CELERY_CFG=./conf/celery_cfg.json
FROM python:3.8.7-slim-buster
################################################################################
# MISE A JOUR DE L'OS
################################################################################
RUN rm -rf /var/lib/apt/lists/* && \
apt-get -y update && \
apt-get -y upgrade && \
apt-get -y purge --auto-remove -o APT::AutoRemove::RecommendsImportant=false
################################################################################
# CREATION UTILISATEUR NON-PRIVILEGIE
################################################################################
RUN addgroup --system --gid 101 nginx && \
adduser --system --disabled-login --ingroup nginx --no-create-home --home /nonexistent --gecos "nginx user" --shell /bin/false --uid 101 nginx
################################################################################
# INSTALLATION NGINX NON-PRIVILEGIE
################################################################################
RUN apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates && \
NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
found=''; \
for server in \
ha.pool.sks-keyservers.net \
hkp://keyserver.ubuntu.com:80 \
hkp://p80.pool.sks-keyservers.net:80 \
pgp.mit.edu \
; do \
echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
done; \
test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* && \
echo "deb http://nginx.org/packages/debian buster nginx" | tee /etc/apt/sources.list.d/nginx.list && \
apt-get update && \
apt-get -y install nginx=1.18.0-1~buster git && \
ln -sf /dev/stdout /var/log/nginx/access.log && \
ln -sf /dev/stderr /var/log/nginx/error.log
RUN chown -R 101:0 /var/cache/nginx && \
chmod -R g+w /var/cache/nginx && \
chown -R 101:0 /etc/nginx && \
chmod -R g+w /etc/nginx
################################################################################
# DEPENDANCES PYTHON
################################################################################
RUN apt-get install --no-install-recommends --no-install-suggests -y \
python-pip \
build-essential gettext \
procps iputils-ping telnet iptables sqlite3 libsqlite3-dev && \
pip install --upgrade pip && \
pip install uwsgi && \
apt-get remove --purge --auto-remove -y \
python-pip \
build-essential
WORKDIR /opt
RUN git clone https://gitlab.ifremer.fr/eo-dataflow/eo-dataflow-ui.git && rm -rf /var/cache/apk/*
#ADD . /opt/eo-dataflow-ui
WORKDIR /opt/eo-dataflow-ui
RUN pip install -r /opt/eo-dataflow-ui/requirements.txt && \
django-admin compilemessages && \
pip install .
################################################################################
# RESSOURCES SUPPLEMENTAIRES
################################################################################
ADD "resources/bin/entrypoint.sh" /opt/entrypoint.sh
ADD "resources/nginx/nginx.conf" /tmp/nginx.conf
ADD "resources/nginx/downloader.conf" /tmp/downloader.conf
ADD "resources/nginx/uwsgi.ini" /opt/eo-dataflow-ui/conf/uwsgi.ini
RUN chgrp -R nginx /opt/entrypoint.sh && \
chmod 555 /opt/entrypoint.sh && \
cat /tmp/downloader.conf > /etc/nginx/conf.d/default.conf && \
cat /tmp/nginx.conf > /etc/nginx/nginx.conf && \
chmod 666 /opt/eo-dataflow-ui/conf/uwsgi.ini
#############################################################################
# CONFIGURATION DOWNLOADER
#############################################################################
ENV PWD=/opt/eo-dataflow-ui
ENV CELERY_CFG=/opt/eo-dataflow-ui/conf/celery_cfg.json
ENV LOGTOFILE=NO
ENV MANAGEMENT_DSN=https://rabbitmq-management-val.ifremer.fr:15672
ENV BROKER_DSN="amqp://cersat-downloader-ui:Bruce and the Spatules@rabbitmq-cluster-val.ifremer.fr:5672/cersat-downloader"
RUN mkdir -p ${PWD}/conf/db && \
export CELERY_CFG=${PWD}/conf/celery_cfg.json && \
django-admin migrate --settings conf.settings --pythonpath $PWD && \
chmod -R 777 ${PWD}/conf/db && \
mkdir -p ${PWD}/conf/export/logs && \
chmod 777 ${PWD}/conf/export/logs && \
ln -s ${PWD}/conf/export/logs ${PWD}/conf/logs
mkdir -p ${PWD}/conf/uwsgi && \
chmod -R 777 ${PWD}/conf/uwsgi && \
chmod 666 /etc/nginx/conf.d/default.conf && \
mkdir -p ${PWD}/conf/media && \
chmod 777 ${PWD}/conf/media && \
mkdir /opt/logs && \
chmod 777 /opt/logs
VOLUME ["/opt/logs"]
#############################################################################
# PORTS POUVANT ETRE EXPOSES
#############################################################################
RUN cp -r ${PWD}/conf/export /home
EXPOSE 8080/tcp
#------------------------------------------------------------------
#############################################################################
# POINT D'ENTREE
#############################################################################
EXPOSE 80
STOPSIGNAL SIGTERM
VOLUME ${PWD}/conf/export
USER 101
ENV URL_PREFIX= SERVER_NAME=
ENV LOG_LEVEL= LOG_TERMINAL=--log-to-terminal
WORKDIR /opt/eo-dataflow-ui/conf
ENTRYPOINT [ "/bin/bash", "scripts/run.sh" ]
ENTRYPOINT ["/bin/bash", "/opt/entrypoint.sh"]
CMD [ "/bin/bash" ]
CMD ["nginx", "-g", "daemon off;"]
include *.txt
recursive-include lib/downloader_django/locale *
recursive-include lib/downloader_django/static *
recursive-include lib/downloader_django/templates *
recursive-include lib/eo_dataflow_ui/locale *
recursive-include lib/eo_dataflow_ui/static *
recursive-include lib/eo_dataflow_ui/templates *
include settings.py
recursive-include downloader *.py
# Downloader UI
# Earth Observation dataflow UI
Web UI for monitoring and administering the downloader daemon
Web UI for monitoring and administering the eo-dataflow manager
# Installation
......@@ -11,9 +11,9 @@ Web UI for monitoring and administering the downloader daemon
conda env create -f environment.yaml
# Activate created env
conda activate downloader_ui
conda activate eo-dataflow-ui
# Install the downloader_ui
# Install the eo-dataflow-ui
./setup.py install
```
......@@ -55,7 +55,7 @@ scripts/run.sh
### docker
```
docker run -p #:80 -e LOG_LEVEL=# -e SERVER_NAME=# -e LOG_TERMINAL=--log-to-terminal -v #:/home/downloader_ui/conf/export downloader_ui
docker run -p #:80 -e LOG_LEVEL=# -e SERVER_NAME=# -e LOG_TERMINAL=--log-to-terminal -v #:/home/eo-dataflow-ui/conf/export eo-dataflow-ui
```
# Development
......
export/celery_cfg.json
\ No newline at end of file
{ "broker_heartbeat": 30,
"imports": [],
"result_backend": "rpc://",
"result_expires": 300,
"task_serializer": "json",
"result_serializer": "json",
"timezone": "Europe/Paris",
"enable_utc": "True",
"result_compression": "bzip2",
"worker_log_color": "False",
"accept_content": ["json"],
"auto_delete": "True",
"expires": 15.0,
"exclusive": "True"
}
{ "management_url" : "https://rabbitmq-management-val.ifremer.fr:15672",
"broker_url": "amqp://cersat-downloader-ui:Bruce and the Spatules@rabbitmq-cluster-val.ifremer.fr:5672/cersat-downloader",
"broker_heartbeat": 30,
"imports": [],
"result_backend": "amqp",
"result_expires": 300,
"task_serializer": "json",
"result_serializer": "json",
"timezone": "Europe/Paris",
"enable_utc": "True",
"result_compression": "bzip2",
"worker_log_color": "False",
"accept_content": ["json"],
"auto_delete": "True",
"expires": 15.0,
"exclusive": "True"
}
# Django settings for django_downloader project.
from __future__ import with_statement
from pkg_resources import resource_filename as pkg_file
from lxml import etree, objectify
import django
import os
from celery import Celery
import simplejson
#PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
PROJECT_ROOT = os.path.dirname(__file__)
# Get settings from environment variables
# -----------------------------------------------------------------------------
CELERY_CFG_FILE = os.environ.get('CELERY_CFG', os.path.join(PROJECT_ROOT, 'celery_cfg.json'))
if not os.path.exists(CELERY_CFG_FILE):
raise Exception('Celery configuration file %s not found.' % CELERY_CFG_FILE)
URL_PREFIX = os.environ.get('URL_PREFIX', '')
DEBUG = False
ADMINS = (
# ('Your Name', 'your_email@example.com'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': os.path.join(PROJECT_ROOT, 'db/downloader.db'), # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
'USER': '',
'PASSWORD': '',
'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
'PORT': '', # Set to empty string for default.
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['.ifremer.fr']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'UTC'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True
# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
#MEDIA_ROOT = WORKSPACE
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = os.path.join(os.path.dirname(__file__), 'static') + '/'
# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/%sstatic/' % URL_PREFIX
# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
pkg_file('downloader_django', 'static'),
)
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
LOCALE_PATHS = (
os.path.join(STATIC_ROOT, 'locale'),
)
# Make this unique, and don't share it with anybody.
SECRET_KEY = 't25jsr7eh9!it09qhw4!^%zg)0&)l3#g^!pyj!#0w+^@2*3x9h'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
#'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
# ...
],
#'builtins': ["debugtools.templatetags.debugtools_tags"],
},
},
]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
# Uncomment the next line for simple clickjacking protection:
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
# Lien vers la page si l'authentification echoue
LOGIN_URL = '%saccount/login/' % URL_PREFIX
# Lien vers la page si l'authentification reussi
LOGIN_REDIRECT_URL = '%sdownloaderadmin/' % URL_PREFIX
ROOT_URLCONF = 'downloader_django.urls'
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'downloader_django.wsgi.application'
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
#'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'downloader_django',
'downloader_django.apps.downloader_admin',
'mod_wsgi.server' if 'NEED_MODWSGI' in os.environ and os.environ.get('NEED_MODWSGI') == "YES" else 'downloader_django.apps'
)
SESSION_ENGINE = "django.contrib.sessions.backends.db"
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
}
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
}
}
#infile_path = os.path.join(APPDATA, 'controller/sc_path.xml')
# with open(infile_path, 'r') as infile:
# paths = objectify.parse(infile).getroot()
# This loads the celery configuration from FELYX_CELERY_CFG file
#with open(os.environ['CELERY_CFG'], 'rb') as f:
with open(CELERY_CFG_FILE, 'rb') as f:
CELERY_CONF = simplejson.loads(f.read())
CELERY = Celery()
CELERY.conf.update(**CELERY_CONF)
export/settings.py
\ No newline at end of file
# Django settings for eo-dataflow-ui project.
from __future__ import with_statement
from pkg_resources import resource_filename as pkg_file
from lxml import etree, objectify
import django
import os
from celery import Celery
import simplejson
PROJECT_ROOT = os.path.dirname(__file__)
# Get settings from environment variables
# -----------------------------------------------------------------------------
BROKER_URL = os.environ.get('BROKER_DSN')
MANAGEMENT_URL = os.environ.get('MANAGEMENT_DSN')
CELERY_CFG_FILE = os.environ.get('CELERY_CFG', os.path.join(PROJECT_ROOT, 'celery_cfg.json'))
if not os.path.exists(CELERY_CFG_FILE):
raise Exception('Celery configuration file %s not found.' % CELERY_CFG_FILE)
URL_PREFIX = os.environ.get('URL_PREFIX', '')
DEBUG = False
ADMINS = (
# ('Your Name', 'your_email@example.com'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': os.path.join(PROJECT_ROOT, 'db/downloader.db'), # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
'USER': '',
'PASSWORD': '',
'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
'PORT': '', # Set to empty string for default.
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
ALLOWED_HOSTS = ['.ifremer.fr']
TIME_ZONE = 'UTC'
LANGUAGE_CODE = 'en'
SITE_ID = 1
USE_I18N = True
USE_L10N = True
USE_TZ = True
MEDIA_ROOT = os.path.join(os.path.dirname(__file__), 'media') + '/'
MEDIA_URL = '/%smedia/' % URL_PREFIX
STATIC_ROOT = os.path.join(os.path.dirname(__file__), 'static') + '/'
STATIC_URL = '/%sstatic/' % URL_PREFIX
STATICFILES_DIRS = (
pkg_file('eo_dataflow_ui', 'static'),
)
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
LOCALE_PATHS = (
os.path.join(STATIC_ROOT, 'locale'),
)
# Make this unique, and don't share it with anybody.
SECRET_KEY = 't25jsr7eh9!it09qhw4!^%zg)0&)l3#g^!pyj!#0w+^@2*3x9h'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
#'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
# ...
],
#'builtins': ["debugtools.templatetags.debugtools_tags"],
},
},
]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
# Uncomment the next line for simple clickjacking protection:
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
# Lien vers la page si l'authentification echoue
LOGIN_URL = '%saccount/login/' % URL_PREFIX
# Lien vers la page si l'authentification reussi
LOGIN_REDIRECT_URL = '%sdownloaderadmin/' % URL_PREFIX
ROOT_URLCONF = 'eo_dataflow_ui.urls'
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'eo_dataflow_ui.wsgi.application'
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
#'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'eo_dataflow_ui',
'eo_dataflow_ui.apps.downloader_admin',
'mod_wsgi.server' if 'NEED_MODWSGI' in os.environ and os.environ.get('NEED_MODWSGI') == "YES" else 'eo_dataflow_ui.apps'
)
SESSION_ENGINE = "django.contrib.sessions.backends.db"
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
}
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},