652 lines
22 KiB
Python
652 lines
22 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
##########################################################################
|
|
# creole.server - distribute creole variables through REST API
|
|
# Copyright © 2012,2013 Pôle de compétences EOLE <eole@ac-dijon.fr>
|
|
#
|
|
# License CeCILL:
|
|
# * in french: http://www.cecill.info/licences/Licence_CeCILL_V2-fr.html
|
|
# * in english http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
|
|
##########################################################################
|
|
|
|
"""Distribute Creole configuration through REST API
|
|
|
|
Setup a daemon based on `cherrypy` listening by default on
|
|
127.0.0.1:8000 for queries on Creole configuration.
|
|
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import threading
|
|
|
|
from creole import eosfunc
|
|
|
|
from traceback import format_exc
|
|
|
|
from os.path import basename, dirname, isdir, samefile, splitext
|
|
|
|
from pyeole.log import init_logging, getLogger
|
|
from pyeole import scriptargs
|
|
|
|
from .config import configeoldir, eoledirs, eoleextradico, \
|
|
eoleextraconfig
|
|
from .loader import creole_loader, load_config_eol, load_extras
|
|
|
|
from .i18n import _
|
|
|
|
from tiramisu.config import Config, SubConfig, undefined
|
|
from tiramisu.error import PropertiesOptionError
|
|
|
|
from pyeole.cherrypy_plugins import InotifyMonitor
|
|
|
|
import cherrypy
|
|
import socket
|
|
|
|
from pyinotify import ProcessEvent
|
|
from pyinotify import IN_DELETE
|
|
from pyinotify import IN_CREATE
|
|
from pyinotify import IN_MODIFY
|
|
from pyinotify import IN_MOVED_TO
|
|
from pyinotify import IN_MOVED_FROM
|
|
|
|
from systemd import daemon
|
|
|
|
import logging
|
|
|
|
# Global logger
|
|
log = getLogger(__name__)
|
|
|
|
lock = threading.Lock()
|
|
|
|
num_error = [(PropertiesOptionError, 1), (KeyError, 2),
|
|
(AttributeError, 4), (Exception, 3)]
|
|
|
|
# For pyinotify handler and filtering
|
|
_INOTIFY_EOL_DIRS = [configeoldir, eoleextraconfig]
|
|
|
|
_INOTIFY_MASK = IN_DELETE | IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_MOVED_FROM
|
|
|
|
|
|
def _inotify_filter(event):
|
|
"""Check if the path must be excluded from being watched.
|
|
|
|
:param event: event to look for
|
|
:type event: :class:`pyinotify.Event`
|
|
:return: if the :data:`event` must be excluded
|
|
:rtype: `bool`
|
|
|
|
"""
|
|
|
|
_INOTIFY_EOL = True
|
|
|
|
if isdir(event.pathname):
|
|
# Always ok for EOLE directories
|
|
for directory in _INOTIFY_EOL_DIRS:
|
|
if not os.access(directory, os.F_OK):
|
|
continue
|
|
if samefile(event.pathname, directory):
|
|
_INOTIFY_EOL = False
|
|
|
|
if not _INOTIFY_EOL:
|
|
return {"EOL": _INOTIFY_EOL}
|
|
|
|
extension = splitext(event.name)[1]
|
|
|
|
if event.mask != IN_DELETE and not os.access(event.pathname, os.F_OK):
|
|
log.debug(_(u'File not accessible: {0}').format(event.pathname))
|
|
return {"EOL": True}
|
|
|
|
if event.mask != IN_DELETE and os.stat(event.pathname).st_size == 0:
|
|
log.debug(_(u'File with null size: {0}').format(event.pathname))
|
|
return {"EOL": True}
|
|
|
|
# Check only for files in EOLE directories
|
|
|
|
for directory in _INOTIFY_EOL_DIRS:
|
|
if not os.access(directory, os.F_OK):
|
|
continue
|
|
if samefile(event.path, directory) or str(event.path).startswith(directory):
|
|
_INOTIFY_EOL = extension != '.eol'
|
|
break
|
|
|
|
return {"EOL": _INOTIFY_EOL}
|
|
|
|
|
|
class CreoleInotifyHandler(ProcessEvent):
|
|
"""Process inotify events
|
|
|
|
"""
|
|
|
|
_server = None
|
|
"""Instance of :class:`CreoleServer`.
|
|
|
|
"""
|
|
|
|
def my_init(self, server):
|
|
"""Subclass constructor.
|
|
|
|
This is the constructor, it is automatically called from
|
|
:meth:`ProcessEvent.__init__()`,
|
|
|
|
Extra arguments passed to ``__init__()`` would be delegated
|
|
automatically to ``my_init()``.
|
|
|
|
"""
|
|
self._server = server
|
|
|
|
def process_default(self, event):
|
|
"""Reload :class:`CreoleServer` on all managed inotify events
|
|
|
|
"""
|
|
inotify_data = _inotify_filter(event)
|
|
if not inotify_data["EOL"]:
|
|
log.warn(_(u'Reload config.eol due to {0} on {1}').format(event.maskname,
|
|
event.pathname))
|
|
try:
|
|
self._server.reload_eol()
|
|
except:
|
|
pass
|
|
else:
|
|
log.debug(_(u'Filtered inotify event for {0}').format(event.pathname))
|
|
|
|
|
|
class CreoleServer(object):
|
|
"""Cherrypy application answering REST requests
|
|
"""
|
|
|
|
def __init__(self, running=True):
|
|
"""Initialize the server
|
|
|
|
Load the tiramisu configuration.
|
|
|
|
:param `bool` running: Is the web server running during server
|
|
initialization.
|
|
|
|
"""
|
|
|
|
log.debug(_(u"Loading tiramisu configuration"))
|
|
self.config = None
|
|
self.reload_config(running)
|
|
|
|
@cherrypy.expose
|
|
@cherrypy.tools.json_out()
|
|
def reload_config(self, running=True):
|
|
lock.acquire()
|
|
|
|
if running:
|
|
# Tell systemd that we are reloading the configuration
|
|
daemon.notify('RELOADING=1')
|
|
|
|
try:
|
|
log.debug(u"Set umask to 0022")
|
|
os.umask(0022)
|
|
reload(eosfunc)
|
|
eosfunc.load_funcs(force_reload=True)
|
|
self.config = creole_loader(load_extra=True, reload_config=False,
|
|
disable_mandatory=True, owner='creoled',
|
|
try_upgrade=False)
|
|
if log.isEnabledFor(logging.DEBUG) and self.config.impl_get_information('load_error', False):
|
|
msg = _('Load creole configuration with errors')
|
|
log.debug(msg)
|
|
ret = self.response()
|
|
|
|
except Exception, err:
|
|
# Avoid using format as exception message could be undecoded
|
|
msg = _('Unable to load creole configuration: ')
|
|
msg += unicode(str(err), 'utf-8')
|
|
if log.isEnabledFor(logging.DEBUG):
|
|
log.debug(msg, exc_info=True)
|
|
else:
|
|
log.error(msg)
|
|
|
|
#self.config = None
|
|
ret = self.response(status=3)
|
|
|
|
if running:
|
|
# Tell systemd that we are now ready again
|
|
daemon.notify('READY=1')
|
|
|
|
lock.release()
|
|
|
|
return ret
|
|
|
|
@cherrypy.expose
|
|
@cherrypy.tools.json_out()
|
|
def reload_eol(self):
|
|
if not self.config:
|
|
return self.reload_config()
|
|
|
|
lock.acquire()
|
|
|
|
# Tell systemd that we are reloading the configuration
|
|
daemon.notify(u'RELOADING=1')
|
|
|
|
config = Config(self.config.cfgimpl_get_description())
|
|
try:
|
|
load_config_eol(config)
|
|
except Exception, err:
|
|
# Avoid using format as exception message could be undecoded
|
|
msg = _('Unable to load creole configuration from config.eol: ')
|
|
msg += unicode(str(err), 'utf-8')
|
|
if log.isEnabledFor(logging.DEBUG):
|
|
log.debug(msg, exc_info=True)
|
|
else:
|
|
log.error(msg)
|
|
|
|
#self.config = None
|
|
ret = self.response(status=3)
|
|
try:
|
|
load_extras(config)
|
|
except:
|
|
msg = _('Unable to load creole configuration from extra: ')
|
|
msg += unicode(str(err), 'utf-8')
|
|
if log.isEnabledFor(logging.DEBUG):
|
|
log.debug(msg, exc_info=True)
|
|
else:
|
|
log.error(msg)
|
|
|
|
#self.config = None
|
|
ret = self.response(status=3)
|
|
else:
|
|
config.read_only()
|
|
self.config = config
|
|
ret = self.response()
|
|
|
|
|
|
# Tell systemd that we are now ready again
|
|
daemon.notify(u'READY=1')
|
|
|
|
lock.release()
|
|
|
|
return ret
|
|
|
|
@cherrypy.expose
|
|
@cherrypy.tools.json_out()
|
|
def valid_mandatory(self):
|
|
if self.config is None:
|
|
return self._no_config()
|
|
try:
|
|
msg = _(u'All variables are not set, please configure your system:')
|
|
error = False
|
|
mandatory_errors = set(self.config.cfgimpl_get_values().mandatory_warnings(force_permissive=True))
|
|
if mandatory_errors != set():
|
|
error = True
|
|
msg += ' ' + _('variables are mandatories') + ' (' + ', '.join(mandatory_errors) + ')'
|
|
force_vars = set()
|
|
for force_store_var in self.config.impl_get_information('force_store_vars'):
|
|
if force_store_var not in mandatory_errors:
|
|
try:
|
|
getattr(self.config, force_store_var)
|
|
force_vars.add(force_store_var)
|
|
except:
|
|
pass
|
|
if force_vars != set():
|
|
error = True
|
|
msg += ' ' + _('variables must be in config file') + ' (' + ', '.join(force_vars) + ')'
|
|
|
|
if error:
|
|
log.debug(mandatory_errors)
|
|
return self.response(msg, 3)
|
|
except Exception, err:
|
|
log.debug(err, exc_info=True)
|
|
return self.response(str(err), 3)
|
|
return self.response()
|
|
|
|
@staticmethod
|
|
def response(response='OK', status=0):
|
|
"""Generate a normalized response
|
|
|
|
:param response: message of the response
|
|
:type response: `object`
|
|
:param status: status code for the response, ``0`` for OK
|
|
:type status: `int`
|
|
:return: response of the form: ``{"status": `int`, "response": `message`}``
|
|
:rtype: `dict`
|
|
|
|
"""
|
|
return {u'status': status, u'response': response}
|
|
|
|
@cherrypy.expose
|
|
@cherrypy.tools.json_out()
|
|
def get(self, *args, **kwargs):
|
|
"""Return the content of a tiramisu path
|
|
|
|
:param args: path elements of the query
|
|
:type args: `list`
|
|
:return: Value of a single variable or sub tree
|
|
:rtype: `dict`
|
|
|
|
"""
|
|
def _remove_properties_error(val):
|
|
new_val = []
|
|
for v in val:
|
|
if isinstance(v, PropertiesOptionError):
|
|
new_val.append({'err': str(v)})
|
|
else:
|
|
new_val.append(v)
|
|
return new_val
|
|
|
|
if self.config is None:
|
|
return self._no_config()
|
|
try:
|
|
config = self.config
|
|
if len(args) != 0:
|
|
subconfig = getattr(config, '.'.join(args))
|
|
else:
|
|
subconfig = config
|
|
if isinstance(subconfig, SubConfig):
|
|
if u'variable' in kwargs:
|
|
name = kwargs[u'variable']
|
|
path = subconfig.find_first(byname=name,
|
|
type_=u'path',
|
|
check_properties=False)
|
|
try:
|
|
val = getattr(config, path)
|
|
except PropertiesOptionError as err:
|
|
if err.proptype == ['mandatory']:
|
|
raise Exception(_(u'Mandatory variable {0} '
|
|
u'is not set.').format(name))
|
|
raise err
|
|
if isinstance(val, list):
|
|
val = _remove_properties_error(val)
|
|
return self.response(val)
|
|
else:
|
|
withoption = kwargs.get(u'withoption')
|
|
withvalue = kwargs.get(u'withvalue')
|
|
if withvalue is None:
|
|
withvalue = undefined
|
|
dico = subconfig.make_dict(withoption=withoption, withvalue=withvalue)
|
|
for key, val in dico.items():
|
|
if isinstance(val, list):
|
|
dico[key] = _remove_properties_error(val)
|
|
return self.response(dico)
|
|
else:
|
|
#if config is a value, not a SubConfig
|
|
if isinstance(subconfig, list):
|
|
subconfig = _remove_properties_error(subconfig)
|
|
return self.response(subconfig)
|
|
except Exception, err:
|
|
log.debug(err, exc_info=True)
|
|
for error_match in num_error:
|
|
if isinstance(err, error_match[0]):
|
|
break
|
|
return self.response(str(err), error_match[1])
|
|
|
|
@cherrypy.expose
|
|
@cherrypy.tools.json_out()
|
|
def list(self, *args):
|
|
"""List subtree pointed by :data:`args`
|
|
|
|
List the nodes and variables under a path.
|
|
|
|
If the path point to a single variable, then return its value.
|
|
|
|
:param args: path elements of the query
|
|
:type args: `list`
|
|
|
|
:return: Nodes and/or variables under a path, or value of a
|
|
variable
|
|
:rtype: `list`
|
|
|
|
"""
|
|
if self.config is None:
|
|
return self._no_config()
|
|
try:
|
|
config = self.config
|
|
if len(args) == 0:
|
|
# root of configuration
|
|
obj = config
|
|
else:
|
|
# Path to a sub configuration
|
|
base = '.'.join(args)
|
|
obj = getattr(config, base)
|
|
if isinstance(obj, SubConfig):
|
|
# Path is a node
|
|
groups = [u'%s/' % g[0] for g in obj.iter_groups()]
|
|
items = [u'%s' % i[0] for i in obj]
|
|
return self.response(groups + items)
|
|
else:
|
|
# Path is a leaf
|
|
value = self.get(*args)[u'response']
|
|
return self.response([value])
|
|
except Exception, err:
|
|
log.debug(err, exc_info=True)
|
|
for error_match in num_error:
|
|
if isinstance(err, error_match[0]):
|
|
break
|
|
return self.response(str(err), error_match[1])
|
|
|
|
def _no_config(self):
|
|
"""Return an error message when no configuration is loaded
|
|
|
|
:return: a failure response
|
|
:rtype: `dict`
|
|
|
|
"""
|
|
return self.response(_(u'No configuration'), status=3)
|
|
|
|
class CreoleDaemon(object):
|
|
"""Run the CreoleServer
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Initialize the cherrypy daemon
|
|
"""
|
|
|
|
# Built-in configuration
|
|
self.argparse = self._load_argparse()
|
|
# Read command line arguments
|
|
self.option = self.argparse.parse_args()
|
|
if self.option.verbose:
|
|
self.option.log_level = u'info'
|
|
if self.option.debug:
|
|
self.option.log_level = u'debug'
|
|
self._configure_log()
|
|
|
|
def _load_argparse(self):
|
|
"""Parse command line arguments
|
|
|
|
:return: command line parser
|
|
:rtype: `argparse.ArgumentParser`
|
|
|
|
"""
|
|
parser = argparse.ArgumentParser(description=u'Run creole daemon',
|
|
parents=[scriptargs.logging('warning')],
|
|
conflict_handler='resolve')
|
|
parser.add_argument("-b", "--base-dir", default='/tmp',
|
|
help=_(u"Base directory in which the server"
|
|
" is launched (default: /tmp)"))
|
|
parser.add_argument("-c", "--conf-file",
|
|
default='/etc/eole/creoled.conf',
|
|
help=_(u"Configuration file of the server"
|
|
" (default: /etc/eole/creoled.conf"))
|
|
parser.add_argument("-d", "--daemon", action='store_true',
|
|
help=_(u"Run the server as a daemon (default: false)"))
|
|
parser.add_argument("-l", "--listen", action='store',
|
|
default='127.0.0.1:8000',
|
|
help=_(u"Listen on the specified IP:PORT"
|
|
" (default: 127.0.0.1:8000)"))
|
|
parser.add_argument("-m", "--mount-base", default='/',
|
|
help=_(u"Base under which the application is mounted"
|
|
" (default: /)"))
|
|
parser.add_argument("-p", "--pidfile",
|
|
default='/tmp/{0}.pid'.format(
|
|
basename(sys.argv[0])),
|
|
help=_(u"Base under which the application is mounted"
|
|
" (default: /)"))
|
|
parser.add_argument("-u", "--user", default='nobody',
|
|
help=_(u"User of the running process"
|
|
" (default: nobody)"))
|
|
parser.add_argument("-g", "--group", default='nogroup',
|
|
help=_(u"Group of the running process"
|
|
" (default: nogroup)"))
|
|
parser.add_argument("--umask", default='0640',
|
|
help=_(u"Umask of the running process"
|
|
" (default: 0644)"))
|
|
return parser
|
|
|
|
def _get_conf(self, name):
|
|
"""Map command line arguments to cherrypy configuration
|
|
|
|
:param name: internal name of argparse option store
|
|
:returns: piece of cherrypy configuration
|
|
:rtype: `dict`
|
|
"""
|
|
try:
|
|
option_map = { 'listen' :
|
|
{ 'server.socket_host' :
|
|
self.option.listen.split(':')[0],
|
|
'server.socket_port' :
|
|
int(self.option.listen.split(':')[1])},
|
|
}
|
|
return option_map[name]
|
|
except KeyError:
|
|
return {}
|
|
|
|
def load_conf(self):
|
|
"""Load daemon configuration
|
|
|
|
Take care to load the configuration in proper order and avoid
|
|
overriding configuration file parameter by default command
|
|
line arguments.
|
|
|
|
Order is:
|
|
|
|
- default values from command line option parser
|
|
|
|
- option from a configuration file
|
|
|
|
- command line arguments
|
|
|
|
"""
|
|
# Load all default value
|
|
config = {'engine.autoreload.on': False}
|
|
for opt in vars(self.option):
|
|
config.update(self._get_conf(opt))
|
|
|
|
cherrypy.config.update( { 'global' : config} )
|
|
|
|
# Load configuration file
|
|
if os.access(self.option.conf_file, os.F_OK):
|
|
cherrypy.config.update(self.option.conf_file)
|
|
|
|
# Override config file option present on command line
|
|
config = {}
|
|
for opt in sys.argv[1:]:
|
|
config.update(self._get_conf(opt))
|
|
cherrypy.config.update( {'global' : config } )
|
|
|
|
def _configure_log(self):
|
|
"""Configure the module logger
|
|
|
|
Avoid logging apache style time since the logger does it.
|
|
|
|
"""
|
|
global log
|
|
log_filename = None
|
|
if self.option.daemon:
|
|
log_filename = u'/var/log/creoled.log'
|
|
|
|
log = init_logging(name=u'creoled', as_root=True,
|
|
level=self.option.log_level,
|
|
console=not self.option.daemon,
|
|
syslog=None,
|
|
filename=log_filename)
|
|
|
|
# Cherrypy do not handle logs
|
|
cherrypy.log.error_file = None
|
|
cherrypy.log.access_file = None
|
|
# Do not output on screen
|
|
cherrypy.log.screen = False
|
|
# Hack to avoid time in log message
|
|
cherrypy.log.time = lambda : ''
|
|
|
|
def run(self):
|
|
"""Start the cherrypy server.
|
|
"""
|
|
engine = cherrypy.engine
|
|
|
|
# Load server but we are not running now
|
|
# Do not let him tell systemd otherwise
|
|
server = CreoleServer(running=False)
|
|
|
|
inotify_handler = CreoleInotifyHandler(server=server)
|
|
|
|
if hasattr(engine, "signal_handler"):
|
|
engine.signal_handler.subscribe()
|
|
# Error exit on SIGINT (Ctl-c) #6177
|
|
engine.signal_handler.set_handler(2, self._kill)
|
|
|
|
if hasattr(engine, "console_control_handler"):
|
|
engine.console_control_handler.subscribe()
|
|
|
|
cherrypy.tree.mount(server, self.option.mount_base,
|
|
config={'global' : {} })
|
|
|
|
# Merge configuration from build-in, configuration file and command line
|
|
self.load_conf()
|
|
|
|
if server.config is None:
|
|
msg = _(u"No configuration found: do not check for container mode.")
|
|
log.warn(msg)
|
|
elif server.config.creole.general.mode_conteneur_actif == 'oui':
|
|
container_ip = server.config.creole.containers.adresse_ip_br0
|
|
container_port = cherrypy.config.get('server.socket_port')
|
|
# Start a server for containers if ip can be bounded
|
|
try:
|
|
container_socket = socket.socket(socket.AF_INET,
|
|
socket.SOCK_STREAM)
|
|
container_socket.setsockopt(socket.SOL_SOCKET,
|
|
socket.SO_REUSEADDR,
|
|
1)
|
|
container_socket.bind((container_ip, container_port))
|
|
container_socket.close()
|
|
except socket.error, err:
|
|
log.error(_(u"Unable to listen for containers: {0}").format(err))
|
|
else:
|
|
container_server = cherrypy._cpserver.Server()
|
|
container_server.socket_host = container_ip
|
|
container_server.socket_port = container_port
|
|
container_server.subscribe()
|
|
|
|
monitor = InotifyMonitor(engine, inotify_handler)
|
|
monitor.subscribe()
|
|
|
|
monitor.watch.add_watch(_INOTIFY_EOL_DIRS, _INOTIFY_MASK, auto_add=True, rec=True)
|
|
|
|
if self.option.pidfile:
|
|
cherrypy.process.plugins.PIDFile(engine,
|
|
self.option.pidfile).subscribe()
|
|
|
|
if self.option.daemon:
|
|
cherrypy.process.plugins.Daemonizer(engine).subscribe()
|
|
|
|
# Drop priviledges
|
|
cherrypy.process.plugins.DropPrivileges(engine,
|
|
uid = self.option.user,
|
|
gid = self.option.group,
|
|
umask = self.option.umask)
|
|
|
|
# Let's start the CherryPy engine so that
|
|
# everything works
|
|
engine.start()
|
|
|
|
# Tell systemd that we are ready
|
|
daemon.notify(u'READY=1')
|
|
|
|
# Run the engine main loop
|
|
engine.block()
|
|
|
|
@staticmethod
|
|
def _kill():
|
|
"""Exit the server with non zero exit code
|
|
"""
|
|
sys.exit(1)
|
|
|
|
if __name__ == '__main__':
|
|
daemon = CreoleDaemon()
|
|
daemon.run()
|