Commit ef43f257 authored by BODERE's avatar BODERE
Browse files

Dockerisation

parent e70c1b89
#.git
.gitignore
.idea
conf/run
conf/db
conf/logs
conf/uwsgi
conf/uwsgi.ini
conf/nginx.conf
#
#
#
#------------------------------------------------------------------
FROM python:3.7-slim-stretch
SHELL ["/bin/bash", "--login", "-c"]
RUN apt -y update && \
apt -y install \
apache2 apache2-dev apache2-utils \
wget ca-certificates \
make gcc musl-dev libxslt-dev \
git
#------------------------------------------------------------------
RUN cd /home && \
git clone https://gitlab.ifremer.fr/downloader/downloader_ui.git && \
rm -rf /var/cache/apk/*
ENV PWD=/home/downloader_ui
WORKDIR ${PWD}
RUN /usr/local/bin/pip3 install -r requirements.txt \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/pypi \
--index-url https://nexus-test.ifremer.fr/repository/pypi-public-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-public-snapshot/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-private-release/simple \
--extra-index-url https://nexus-test.ifremer.fr/repository/pypi-private-snapshot/simple && \
/usr/local/bin/pip3 install --no-binary "mod_wsgi" mod_wsgi uwsgi && \
/usr/local/bin/pip3 install .
ENV USER_NAME=www-data GROUP_NAME=www-data
ENV CELERY_CFG=./conf/celery_cfg.json
RUN mkdir -p ${PWD}/conf/db && \
export CELERY_CFG=${PWD}/conf/celery_cfg.json && \
django-admin migrate --settings conf.settings --pythonpath $PWD && \
chmod -R 777 ${PWD}/conf/db && \
mkdir -p ${PWD}/conf/export/logs && \
chmod 777 ${PWD}/conf/export/logs && \
ln -s ${PWD}/conf/export/logs ${PWD}/conf/logs
RUN cp -r ${PWD}/conf/export /home
#------------------------------------------------------------------
EXPOSE 80
VOLUME ${PWD}/conf/export
ENV URL_PREFIX= SERVER_NAME=
ENV LOG_LEVEL= LOG_TERMINAL=--log-to-terminal
ENTRYPOINT [ "/bin/bash", "scripts/run.sh" ]
CMD [ "/bin/bash" ]
......@@ -17,6 +17,46 @@ conda activate downloader_ui
./setup.py install
```
## Configuration
```
# Session database
django-admin migrate --settings conf.settings --pythonpath .
```
## Settings / run
```
vi conf/settings.py
vi conf/celery_cfg.json
```
### nginx
```
vi conf/nginx.conf
vi conf/uwsgi.ini
conf/run
```
### apache
```
LOG_LEVEL="--log-level DEBUG
USER_NAME=apache
GROUP_NAME=apache
PORT_NO=80
SERVER_NAME= url
LOG_DIRECTORY=--log-directory ${PWD}/conf/logs/
scripts/run.sh
```
### docker
```
docker run -p #:80 -e LOG_LEVEL=# -e SERVER_NAME=# -e LOG_TERMINAL=--log-to-terminal -v #:/home/downloader_ui/conf/export downloader_ui
```
# Development
......
{ "management_url" : "https://rabbitmq-management-val.ifremer.fr:15672",
"broker_url": "amqp://cersat-downloader-ui:Bruce and the Spatules@rabbitmq-cluster-val.ifremer.fr:5672/cersat-downloader",
"broker_heartbeat": 30,
"imports": [],
"result_backend": "amqp",
"result_expires": 300,
"task_serializer": "json",
"result_serializer": "json",
"timezone": "Europe/Paris",
"enable_utc": "True",
"result_compression": "bzip2",
"worker_log_color": "False",
"accept_content": ["json"],
"auto_delete": "True",
"expires": 15.0,
"exclusive": "True"
}
export/celery_cfg.json
\ No newline at end of file
{ "management_url" : "https://rabbitmq-management-val.ifremer.fr:15672",
"broker_url": "amqp://cersat-downloader-ui:Bruce and the Spatules@rabbitmq-cluster-val.ifremer.fr:5672/cersat-downloader",
"broker_heartbeat": 30,
"imports": [],
"result_backend": "amqp",
"result_expires": 300,
"task_serializer": "json",
"result_serializer": "json",
"timezone": "Europe/Paris",
"enable_utc": "True",
"result_compression": "bzip2",
"worker_log_color": "False",
"accept_content": ["json"],
"auto_delete": "True",
"expires": 15.0,
"exclusive": "True"
}
......@@ -7,19 +7,21 @@ import os
from celery import Celery
import simplejson
#PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
PROJECT_ROOT = os.path.dirname(__file__)
# Get settings from environment variables
# -----------------------------------------------------------------------------
if 'CELERY_CFG' not in os.environ:
raise Exception('Missing CELERY_CFG environment variable.')
if not os.path.exists(os.environ['CELERY_CFG']):
raise Exception('Celery configuration file %s not found.' % os.environ['CELERY_CFG'])
CELERY_CFG_FILE = os.environ.get('CELERY_CFG', os.path.join(PROJECT_ROOT, 'celery_cfg.json'))
if not os.path.exists(CELERY_CFG_FILE):
raise Exception('Celery configuration file %s not found.' % CELERY_CFG_FILE)
URL_PREFIX = os.environ.get('URL_PREFIX', '')
PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
URL_PREFIX = os.environ.get('URL_PREFIX', '')
DEBUG = True
DEBUG = False
ADMINS = (
# ('Your Name', 'your_email@example.com'),
......@@ -30,7 +32,7 @@ MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'db/downloader.db', # Or path to database file if using sqlite3.
'NAME': os.path.join(PROJECT_ROOT, 'db/downloader.db'), # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
'USER': '',
'PASSWORD': '',
......@@ -42,7 +44,7 @@ DATABASES = {
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []
ALLOWED_HOSTS = ['.ifremer.fr']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
......@@ -54,7 +56,6 @@ TIME_ZONE = 'UTC'
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
......@@ -113,7 +114,17 @@ SECRET_KEY = 't25jsr7eh9!it09qhw4!^%zg)0&)l3#g^!pyj!#0w+^@2*3x9h'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
#'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
# ...
],
#'builtins': ["debugtools.templatetags.debugtools_tags"],
},
},
]
......@@ -142,7 +153,7 @@ INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
#'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
# Uncomment the next line to enable the admin:
......@@ -151,9 +162,11 @@ INSTALLED_APPS = (
# 'django.contrib.admindocs',
'downloader_django',
'downloader_django.apps.downloader_admin',
#'downloader_django.apps.monitor',
'mod_wsgi.server' if 'NEED_MODWSGI' in os.environ and os.environ.get('NEED_MODWSGI') == "YES" else 'downloader_django.apps'
)
SESSION_ENGINE = "django.contrib.sessions.backends.db"
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
......@@ -187,28 +200,10 @@ LOGGING = {
# with open(infile_path, 'r') as infile:
# paths = objectify.parse(infile).getroot()
# CONF_PATH= os.path.join(WORKSPACE if paths.downloads_config.get('location') == "workspace" else APPDATA,
# str(paths.downloads_config))
# WORKSPACE_DOWNLOAD= os.path.join(WORKSPACE if paths.work.get('location') == "workspace" else APPDATA,
# str(paths.work))
# LOG_PATH = os.path.join(WORKSPACE if paths.process_log_archive.get('location') == "workspace" else APPDATA,
# str(paths.process_log_archive))
# CACHE_JOB_PATH = os.path.join(WORKSPACE if paths.jobfile.get('location') == "workspace" else APPDATA,
# str(paths.jobfile))
# CACHE_PROVIDER_PATH = os.path.join(WORKSPACE if paths.providerfile.get('location') == "workspace" else APPDATA,
# str(paths.providerfile))
#BIN_PATH= str(paths.exec_path)
#CACHE_SYNCHRO_PATH = os.path.join(WORKSPACE, 'cache/synchrofile/')
#infile_path = os.path.join(APPDATA, 'controller/sc_global.xml')
# with open(infile_path, 'r') as infile:
# global_config = objectify.parse(infile).getroot()
#LOCK_FILE_EXTENSION = global_config.lockfile_extension
# This loads the celery configuration from FELYX_CELERY_CFG file
with open(os.environ['CELERY_CFG'], 'rb') as f:
#with open(os.environ['CELERY_CFG'], 'rb') as f:
with open(CELERY_CFG_FILE, 'rb') as f:
CELERY_CONF = simplejson.loads(f.read())
CELERY = Celery()
CELERY.conf.update(**CELERY_CONF)
# Django settings for django_downloader project.
from __future__ import with_statement
from pkg_resources import resource_filename as pkg_file
from lxml import etree, objectify
import django
import os
from celery import Celery
import simplejson
# Get settings from environment variables
# -----------------------------------------------------------------------------
if 'CELERY_CFG' not in os.environ:
raise Exception('Missing CELERY_CFG environment variable.')
if not os.path.exists(os.environ['CELERY_CFG']):
raise Exception('Celery configuration file %s not found.' % os.environ['CELERY_CFG'])
URL_PREFIX = os.environ.get('URL_PREFIX', '')
PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
DEBUG = True
ADMINS = (
# ('Your Name', 'your_email@example.com'),
)
MANAGERS = ADMINS
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'db/downloader.db', # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
'USER': '',
'PASSWORD': '',
'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
'PORT': '', # Set to empty string for default.
}
}
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ['vcerdownload1-test.ifremer.fr']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'UTC'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True
# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
#MEDIA_ROOT = WORKSPACE
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = os.path.join(os.path.dirname(__file__), 'static') + '/'
# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/%sstatic/' % URL_PREFIX
# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
pkg_file('downloader_django', 'static'),
)
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
LOCALE_PATHS = (
os.path.join(STATIC_ROOT, 'locale'),
)
# Make this unique, and don't share it with anybody.
SECRET_KEY = 't25jsr7eh9!it09qhw4!^%zg)0&)l3#g^!pyj!#0w+^@2*3x9h'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
#'django.template.context_processors.debug',
'django.template.context_processors.request'
# ...
],
#'builtins': ["debugtools.templatetags.debugtools_tags"],
},
},
]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
# Uncomment the next line for simple clickjacking protection:
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)
# Lien vers la page si l'authentification echoue
LOGIN_URL = '%saccount/login/' % URL_PREFIX
# Lien vers la page si l'authentification reussi
LOGIN_REDIRECT_URL = '%sdownloaderadmin/' % URL_PREFIX
ROOT_URLCONF = 'downloader_django.urls'
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'downloader_django.wsgi.application'
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'downloader_django',
'downloader_django.apps.downloader_admin',
#'downloader_django.apps.monitor',
)
SESSION_ENGINE = "django.contrib.sessions.backends.db"
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
}
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
}
}
#infile_path = os.path.join(APPDATA, 'controller/sc_path.xml')
# with open(infile_path, 'r') as infile:
# paths = objectify.parse(infile).getroot()
# This loads the celery configuration from FELYX_CELERY_CFG file
with open(os.environ['CELERY_CFG'], 'rb') as f:
CELERY_CONF = simplejson.loads(f.read())
CELERY = Celery()
CELERY.conf.update(**CELERY_CONF)
export/settings.py
\ No newline at end of file
[uwsgi]
chdir=/export/home/python/applications/downloader_ui/conf
gid=cersat
socket=/export/home/python/applications/test-django/uwsgi/downloader.sock
socket=%(chdir)/uwsgi/downloader.sock
chmod-socket=666
chown-socket=medspi:cersat
python-auto-reload=true
master=true
processes=2
......@@ -15,10 +19,9 @@ threads=40
env=DJANGO_SETTINGS_MODULE=settings
env=URL_PREFIX=dev/downloader/
env=CELERY_CFG=celery_cfg.json
pythonpath=/export/home/python/applications/test-django
pythonpath=%(chdir)
module=downloader_django.wsgi:application
chdir=/export/home/python/applications/test-django
logto=/export/home/python/applications/test-django/logs/downloader.log
logto=%(chdir)/logs/downloader.log
heartbeat=10
......@@ -438,7 +438,7 @@ def reload_from_file(request, xml_filename, download_id):
elif result == "0":
status = "no file found to redownload"
elif result == "1":
status = "%one file found and marked to redownload"
status = "one file found and marked to redownload"
else:
status = "%s files found and marked to redownload" % (result)
......
......@@ -24,6 +24,7 @@ urlpatterns = [
# url(r"^%saccount/logout" % url_prefix, django.contrib.auth.views.logout, {"next_page": "%saccount/login" % url_prefix}, name="logout"),
url(r"%sdownloaderadmin/" % url_prefix, include(dladm_urls, namespace="dladmin")),
url(r"%s" % url_prefix, include(dladm_urls, namespace="dladm")),
]
urlpatterns += staticfiles_urlpatterns()
django~=1.11
lxml~=4.4.1
celery~=4.3.0
#uWSGI~=2.0.18
simplejson~=3.16
requests~=2.21
\ No newline at end of file
requests~=2.21
#!/usr/bin/env bash
if [[ ${LOG_LEVEL} != "" ]]; then
LOG_LEVEL="--log-level ${LOG_LEVEL}"
fi
if [[ ${USER_NAME} == "" ]]; then
USER_NAME=apache
GROUP_NAME=apache
fi
if [[ ${PORT_NO} == "" ]]; then
PORT_NO=80
fi
if [[ ${SERVER_NAME} == "" ]]; then
SERVER_NAME=${HOSTNAME}
fi
if [[ ${LOG_TERMINAL} == "" ]]; then
LOG_DIRECTORY="--log-directory ${PWD}/conf/logs/"
fi
if [[ ! -f ${PWD}/conf/export/settings.py ]]; then
if [[ -f /home/export/settings.py ]]; then
cp -r /home/export ${PWD}/conf
echo STOP : configuration file settings.py has been initialized.\
Please update the settings in this file
else
echo ERROR : settings.py file not found
fi
else
export NEED_MODWSGI=YES
django-admin runmodwsgi \
--settings conf.settings \
--pythonpath ${PWD}/ \
--user ${USER_NAME} --group ${GROUP_NAME} \
--server-name ${SERVER_NAME} \
--port ${PORT_NO} \
${LOG_DIRECTORY} ${LOG_TERMINAL} \
--startup-log --access-log ${LOG_LEVEL}
fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment