You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

767 lines
33 KiB

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

# vim: fileencoding=utf-8
#
from jinja2.ext import Extension
from jinja2 import Environment, FileSystemLoader, TemplateSyntaxError, nodes,\
contextfunction
from jinja2.utils import missing
from jinja2.runtime import Context, Undefined
from collections.abc import MutableMapping
from collections import OrderedDict
from importlib import import_module
import re
import os
from ..utils.package import PackageAtom, PackageAtomError
from ..utils.files import join_paths
# Типы шаблона: директория или файл.
DIR, FILE = range(2)
class IncorrectParameter(Exception):
pass
class DefaultParameterError(Exception):
pass
class ConditionFailed(TemplateSyntaxError):
pass
class TemplateParametersChecker:
'''Класс для хранения, проверки и разбора параметров шаблона.'''
available_parameters = {'name', 'path', 'append', 'chmod', 'chown',
'autoupdate', 'env', 'force', 'source', 'format',
'protected', 'mirror', 'run', 'exec', 'env',
'package', 'merge', 'postmerge', 'action',
'rebuild', 'restart', 'stop', 'start'}
inheritable_parameters = {'chmod': (None, None), 'chown': (None, None),
'autoupdate': (None, None), 'env': (None, None),
'package': (None, None), 'action': (None, None)}
available_appends = set()
directory_default_values = {'chown': 'root:root',
'chmod': '755'}
file_default_values = {'chown': 'root:root',
'chmod': '644'}
available_formats = set()
format_is_inspected = False
chmod_value_regular = re.compile(
r'([r-][w-][x-])([r-][w-][x-])([r-][w-][x-])')
package_atom_parser = PackageAtom()
def __init__(self, parameters_container, template_type, parameters=dict(),
chroot_path='/'):
self.template_type = template_type
self.chroot_path = chroot_path
self._parameters_container = parameters_container
self._inspect_formats_package()
self.checkers_list = OrderedDict({
'package': self.check_package_parameter,
'append': self.check_append_parameter,
'rebuild': self.check_rebuild_parameter,
'restart': self.check_restart_parameter,
'stop': self.check_stop_parameter,
'start': self.check_start_parameter,
'chown': self.check_chown_parameter,
'chmod': self.check_chmod_parameter,
'autoupdate': self.check_autoupdate_parameter,
'source': self.check_source_parameter,
'force': self.check_force_parameter
})
try:
if template_type == DIR:
self.check_template_parameters(self.directory_default_values)
elif template_type == FILE:
self.check_template_parameters(self.file_default_values)
except IncorrectParameter as error:
raise DefaultParameterError('Default parameter value error: {}'.
format(str(error)))
if parameters:
self.check_template_parameters(parameters)
def __getattr__(self, parameter_name):
if parameter_name not in self.available_parameters:
raise IncorrectParameter("Unknown parameter: '{}'".
format(parameter_name))
elif parameter_name not in self._parameters_container:
return False
else:
return self._parameters_container[parameter_name]
def check_template_parameter(self, parameter_name, parameter_value):
# Если параметр наследуем и уже встречался до этого с тем же
# значением -- второй раз не проверяем его.
if (parameter_name in self.inheritable_parameters and
parameter_value == self.inheritable_parameters[parameter_name][0]):
return self.inheritable_parameters[parameter_name][1]
if parameter_name not in self.available_parameters:
raise IncorrectParameter("Unknown parameter '{0}'".
format(parameter_name))
elif parameter_name in self.checkers_list:
checked_value = self.checkers_list[parameter_name](
parameter_value)
if parameter_name in self.inheritable_parameters:
self.inheritable_parameters[parameter_name] =\
(parameter_value, checked_value)
return checked_value
def check_template_parameters(self, parameters):
for parameter_name in parameters:
if (parameter_name in self.inheritable_parameters and
parameters[parameter_name] ==
self.inheritable_parameters[parameter_name][0]):
self._parameters_container[parameter_name] =\
self.inheritable_parameters[parameter_name][1]
continue
if parameter_name not in self.available_parameters:
raise IncorrectParameter("Unknown parameter '{0}'".
format(parameter_name))
elif parameter_name in self.checkers_list:
parameter_value = self.checkers_list[parameter_name](
parameters[parameter_name]
)
self._parameters_container[parameter_name] = parameter_value
if parameter_name in self.inheritable_parameters:
self.inheritable_parameters[parameter_name] =\
(parameter_value, parameters[parameter_name])
def check_package_parameter(self, parameter_value):
try:
self.package_atom_parser.parse_package_parameter(parameter_value)
except PackageAtomError as error:
raise IncorrectParameter(str(error))
parameter_value = self.package_atom_parser.atom_dictionary
return parameter_value
def check_append_parameter(self, parameter_value):
if parameter_value not in self.available_appends:
raise IncorrectParameter("Unacceptable value '{}' of parameter"
" 'append'".format(parameter_value))
return parameter_value
def check_rebuild_parameter(self, parameter_value):
if isinstance(parameter_value, bool):
raise IncorrectParameter("'rebuild' parameter value is not bool")
elif 'package' not in self._parameters_container:
raise IncorrectParameter(("'source' parameter is set without "
"'package' parameter"))
return parameter_value
def check_restart_parameter(self, parameter_value):
if parameter_value and isinstance(parameter_value, str):
return parameter_value
else:
raise IncorrectParameter(
"'restart' parameter value is not correct")
def check_stop_parameter(self, parameter_value):
if parameter_value and isinstance(parameter_value, str):
return parameter_value
else:
raise IncorrectParameter("'stop' parameter value is not correct")
def check_start_parameter(self, parameter_value):
if parameter_value and isinstance(parameter_value, str):
return parameter_value
else:
raise IncorrectParameter("'start' parameter value is not correct")
def check_run_parameter(self, parameter_value):
if parameter_value and isinstance(parameter_value, str):
return parameter_value
else:
raise IncorrectParameter("'run' parameter value is nkt correct")
def check_exec_parameter(self, parameter_value):
if parameter_value and isinstance(parameter_value, str):
return parameter_value
else:
raise IncorrectParameter("'exec' parameter value is not correct")
def check_chown_parameter(self, parameter_value):
if not parameter_value or isinstance(parameter_value, bool):
raise IncorrectParameter("'chown' parameter value is empty.")
parameter_value = self.get_chown_values(parameter_value)
return parameter_value
def check_chmod_parameter(self, parameter_value):
result = self.chmod_value_regular.search(parameter_value)
if result:
parameter_value = ''
for group_number in range(3):
current_group = result.groups()[group_number]
num = ''
for sym_number in range(3):
if current_group[sym_number] != '-':
num = num + '1'
else:
num = num + '0'
parameter_value = parameter_value + num
return int(parameter_value, 2)
elif parameter_value.isdigit():
parameter_value = int(parameter_value, 8)
return parameter_value
else:
raise IncorrectParameter("'chmod' parameter value is not correct")
def check_source_parameter(self, parameter_value):
if self.chroot_path != '/':
real_path = join_paths(self.chroot_path, parameter_value)
else:
real_path = parameter_value
if not parameter_value or isinstance(parameter_value, bool):
raise IncorrectParameter("'source' parameter value is empty")
elif (self.template_type == DIR and
('append' not in self._parameters_container or
self._parameters_container['append'] != 'link')):
raise IncorrectParameter(
("'source' parameter is set without "
"'append = link' for directory template")
)
elif not os.path.exists(real_path):
raise IncorrectParameter(
"File from 'source' parameter does not exist")
return os.path.normpath(real_path)
def check_force_parameter(self, parameter_value):
if isinstance(parameter_value, bool):
return parameter_value
else:
raise IncorrectParameter("'force' parameter value is not bool")
def check_autoupdate_parameter(self, parameter_value):
print('autoupdate value = {}'.format(parameter_value))
if isinstance(parameter_value, bool):
return parameter_value
else:
raise IncorrectParameter(
"'autoupdate' parameter value is not bool")
def get_chown_values(self, chown: str):
"""Получить значения uid и gid из параметра chown."""
if chown and ':' in chown:
user_name, group_name = chown.split(':')
if user_name.isdigit():
uid = int(user_name)
else:
import pwd
try:
if self.chroot_path == '/':
uid = pwd.getpwnam(user_name).pw_uid
else:
uid = self.get_uid_from_passwd(user_name)
except (KeyError, TypeError):
self.output.set_error(
format(user_name))
raise IncorrectParameter(
("'chown' value '{0}' is not correct:"
"no such user in the system: {1}").
format(chown, user_name))
if group_name.isdigit():
gid = int(group_name)
else:
import grp
try:
if self.chroot_path == '/':
gid = grp.getgrnam(group_name).gr_gid
else:
gid = self.get_gid_from_group(group_name)
except (KeyError, TypeError):
raise IncorrectParameter(
("'chown' value '{0}' is not correct:"
"no such group in the system: {1}").
format(chown, group_name))
return {'uid': uid, 'gid': gid}
else:
raise IncorrectParameter("'chown' value '{0}' is not correct".
format(chown, self.template_path))
def get_uid_from_passwd(self, user_name: str):
"""Взять uid из chroot passwd файла."""
passwd_file_path = os.path.join(self.chroot_path, 'etc/passwd')
passwd_dictionary = []
if os.path.exists(passwd_file_path):
with open(passwd_file_path, 'r') as passwd_file:
for line in passwd_file:
if line.startswith('#'):
continue
passwd_item = tuple(line.split(':')[0:3:2])
if (len(passwd_item) > 1 and passwd_item[0]
and passwd_item[0]):
passwd_dictionary.append(passwd_item)
passwd_dictionary = dict(passwd_dictionary)
return int(passwd_dictionary[user_name])
else:
IncorrectParameter("passwd file was not found in {}".
format(passwd_file_path))
def get_gid_from_group(self, group_name: str):
"""Взять gid из chroot group файла."""
group_file_path = os.path.join(self.chroot_path, 'etc/group')
group_dictionary = []
if os.path.exists(group_file_path):
with open(group_file_path, 'r') as group_file:
for line in group_file:
if line.startswith('#'):
continue
group_item = tuple(line.split(':')[0:3:2])
if len(group_item) > 1 and group_item[0] and group_item[1]:
group_dictionary.append(group_item)
group_dictionary = dict(group_dictionary)
if group_name in group_dictionary:
return int(group_dictionary[group_name])
else:
IncorrectParameter("'{0}' gid was not found in {1}".
format(group_name, group_file_path))
else:
IncorrectParameter("group file was not found in {}".
format(group_file_path))
@classmethod
def _inspect_formats_package(cls):
'''Метод для определения множества доступных форматов и
предоставляемых ими параметров.'''
if cls.format_is_inspected:
return
parameters_set = set()
format_set = set()
format_directory_path = os.path.join(os.path.dirname(__file__),
'format')
for module_name in os.listdir(format_directory_path):
if (os.path.isdir(os.path.join('format', module_name)) or
module_name == '__init__.py'):
continue
if module_name.endswith('.py'):
module_name = module_name[:-3]
try:
module = import_module('calculate.templates.format.{}'.
format(module_name))
for obj in dir(module):
if obj.endswith('Format') and obj != 'BaseFormat':
format_class = getattr(module, obj, False)
if format_class:
format_set.add(format_class.FORMAT)
parameters = getattr(format_class,
'FORMAT_PARAMETERS', set())
parameters_set.update(parameters)
except Exception:
continue
cls.available_formats = format_set
cls.available_parameters.update(parameters_set)
cls.formats_inspected = True
class Variables(MutableMapping):
'''Класс заглушка вместо модуля переменных для тестов.'''
def __init__(self, *args, **kwargs):
self.__attrs = dict(*args, **kwargs)
def __next__(self):
iterator = iter(self.__attrs)
return next(iterator)
def __getattribute__(self, name):
if name == '_Variables__attrs':
return super().__getattribute__(name)
try:
return self.__attrs[name]
except KeyError:
raise AttributeError(name)
def __getitem__(self, name):
return self.__attrs[name]
def __setitem__(self, name, value):
self.__attrs[name] = value
def __delitem__(self, name):
del self.__attrs[name]
def __iter__(self):
return iter(self.__attrs)
def __len__(self):
return len(self.__attrs)
def __repr__(self):
return '<Variables {}>'.format(self.__attrs)
def resolve_or_missing(context, key, missing=missing, env={}):
'''Переопределение функции из для поиска значений переменных из jinja2.
Ищет переменные в datavars.'''
datavars = context.parent['__datavars__']
if key in context.vars:
return context.vars[key]
if key in context.parent:
return context.parent[key]
if key in datavars:
return datavars[key]
for name in env:
if name in datavars and key in datavars[name]:
return datavars[name][key]
return missing
class CalculateContext(Context):
'''Класс контекста позволяющий искать значения datavars и сохранять их.'''
_env_set = set()
def resolve(self, key):
if self._legacy_resolve_mode:
rv = resolve_or_missing(self, key,
env=self._env_set)
else:
rv = self.resolve_or_missing(key)
if rv is missing:
return self.environment.undefined(name=key)
return rv
def resolve_or_missing(self, key):
if self._legacy_resolve_mode:
rv = self.resolve(key)
if isinstance(rv, Undefined):
rv = missing
return rv
return resolve_or_missing(self, key,
env=self._env_set)
class ParametersContainer(MutableMapping):
'''Класс для хранения параметров, взятых из шаблона, и передачи
их шаблонизатору.'''
def __init__(self, parameters_dictionary={}):
self.__parameters = parameters_dictionary
self._new_template = True
def set_parameters(self, *args, **kwargs):
parameters = dict(*args, **kwargs)
self.__parameters.update(parameters)
def __getattr__(self, parameter_name):
if (parameter_name not in
TemplateParametersChecker.available_parameters):
raise IncorrectParameter("Unknown parameter: '{}'".
format(parameter_name))
elif parameter_name not in self.__parameters:
return False
else:
return self.__parameters[parameter_name]
def __getitem__(self, name):
return self.__parameters[name]
def __setitem__(self, name, value):
self.__parameters[name] = value
def __delitem__(self, name):
del self.__parameters[name]
def __iter__(self):
return iter(self.__parameters)
def __len__(self):
return len(self.__parameters)
def __repr__(self):
return '<ParametersContainer: {0}>'.format(self.__parameters)
@property
def parameters(self):
return self.__parameters
class CalculateExtension(Extension):
'''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.'''
_parameters_set = set()
_datavars = Variables()
def __init__(self, environment):
print('EXTENSION IS INITIALIZED')
self.tags = {'calculate', 'save', 'set_var'}
self.CONDITION_TOKENS_TYPES = {'eq', 'ne', 'lt', 'gt', 'lteq', 'gteq'}
self.LITERAL_TOKENS_TYPES = {'string', 'integer', 'float'}
self.TARGET_FILES_SET = {'grp', 'system', 'etc', 'local', 'remote'}
self.parse_methods = {'calculate': self.parse_calculate,
'save': self.parse_save}
self.parameters_checker = None
self.new_parameters_set
self.environment = environment
def parse(self, parser):
self.parameters_checker = None
self.parser = parser
self.stream = parser.stream
tag_token = self.stream.current.value
return [self.parse_methods[tag_token]()]
def parse_save(self):
'''Метод для разбора тега save, сохраняющего значение указанной
переменной datavars.'''
lineno = next(self.stream).lineno
target_file = nodes.Const('', lineno=lineno)
if self.stream.skip_if('dot'):
target_file_name = self.stream.expect('name').value
if target_file_name in self.TARGET_FILES_SET:
target_file = nodes.Const(target_file_name)
else:
TemplateSyntaxError("Unknown target file '{}'".
format(target_file_name),
lineno=lineno)
# получаем список из имени переменной.
module_name = self.stream.expect('name').value
variable_name = [nodes.Const(module_name, lineno=lineno)]
while self.stream.skip_if('dot'):
name = self.stream.expect('name').value
variable_name.append(nodes.Const(name, lineno=lineno))
variable_name = nodes.List(variable_name, lineno=lineno)
if self.stream.skip_if('assign'):
right_value = self.parser.parse_expression(with_condexpr=True)
save_variable_node = self.call_method('save_variable',
[variable_name,
right_value,
target_file,
nodes.ContextReference()],
lineno=lineno)
return nodes.Output([save_variable_node], lineno=lineno)
else:
TemplateSyntaxError("'=' is expected in 'save' tag.",
lineno=lineno)
def parse_calculate(self):
'''Метод для разбора тега calculate, содержащего значения параметров и
условия выполнения шаблона.'''
pairs_list = []
expect_comma_flag = False
lineno = next(self.stream).lineno
while self.stream.current.type != 'block_end':
if expect_comma_flag:
self.stream.expect('comma')
if (self.stream.current.type == 'name'
and self.stream.current.value in self._parameters_set
and self.stream.look().type != 'dot'
and self.stream.look().type not in
self.CONDITION_TOKENS_TYPES):
# разбираем параметр.
# pairs_list.append(self.get_parameter_node())
name_node, value_node = self.get_parameter_node()
check_node = self.call_method('check_parameter',
[name_node,
value_node,
nodes.ContextReference()],
lineno=lineno)
pairs_list.append(check_node)
elif (self.stream.current.type == 'name'
or self.stream.current.type == 'lparen'):
# разбираем условие. Если условие False -- кидаем исключение.
condition_result = self.get_condition_result()
if not condition_result:
raise ConditionFailed(
'Condition is failed',
lineno=self.stream.current.lineno
)
else:
raise TemplateSyntaxError('Name is expected in calculate tag.',
lineno=self.stream.current.lineno)
expect_comma_flag = True
# dictionary_node = nodes.Dict(pairs_list)
# save_node = self.call_method('save_parameters',
# [dictionary_node,
# nodes.ContextReference()],
# lineno=lineno)
# return nodes.Output([save_node], lineno=lineno)
return nodes.Output(pairs_list, lineno=lineno)
def check_parameter(self, parameter_name, parameter_value, context):
parameters_object = context.parent['__parameters__']
if not self.parameters_checker:
template_type = context.parent['__template_type__']
chroot_path = context.parent['__datavars__'].main.cl_chroot_path
self.parameters_checker = TemplateParametersChecker(
template_type,
chroot_path=chroot_path
)
parameters_object._new_template = True
elif not parameters_object._new_template:
print('pair = {0}: {1}'.format(parameter_name, parameter_value))
return ''
def get_condition_result(self):
'''Метод для разбора условий из тега calculate.'''
condition_list = []
# собираем исходный код условия из токенов.
# вероятно, следует придумать лучший способ.
while (self.stream.current.type != 'block_end' and
self.stream.current.type != 'comma'):
if self.stream.current.type == 'string':
condition_list.append("'{}'".format(
self.stream.current.value
))
elif self.stream.current.type == 'dot':
self.stream.skip(1)
if self.stream.current.type == 'name':
next_name = '.' + self.stream.current.value
else:
raise TemplateSyntaxError(
'Variable name is not correct.',
lineno=self.stream.current.lineno
)
condition_list[-1] = condition_list[-1] + next_name
else:
condition_list.append(
str(self.stream.current.value)
)
self.stream.skip(1)
condition = ' '.join(condition_list)
# компилируем исходный код условия и получаем результат его вычисления.
cond_expr = self.environment.compile_expression(condition)
condition_result = cond_expr(__datavars__=self._datavars)
return condition_result
def save_variable(self, variable_name, right_value, target_file, context):
'''Метод для сохранения значений переменных указанных в теге save.'''
# временная реализация.
datavars = context.parent['__datavars__']
module_name = variable_name[0]
namespaces = variable_name[1:-1]
variable_name = variable_name[-1]
if module_name in datavars:
variables_module = datavars[module_name]
for namespace in namespaces:
if namespace not in variables_module:
variables_module[namespace] = Variables()
variables_module = variables_module[namespace]
variables_module[variable_name] = right_value
else:
AttributeError("Unknown variables module '{}'".
format(module_name))
return ''
def get_parameter_node(self):
'''Метод для разбра параметров, содержащихся в теге calculate.'''
lineno = self.stream.current.lineno
parameter_name = self.stream.expect('name').value
parameter_name_node = nodes.Const(parameter_name, lineno=lineno)
if self.stream.skip_if('assign'):
parameter_value = self.stream.current.value
parameter_rvalue = self.parser.parse_expression(with_condexpr=True)
if parameter_name == 'env':
# если параметр env -- обновляем множенство значений env
# контекста вo время парсинга.
env_names = parameter_value.split(',')
for name in env_names:
CalculateContext._env_set.add(name.strip())
else:
parameter_rvalue = nodes.Const(True, lineno=lineno)
# return nodes.Pair(parameter_name_node, parameter_rvalue)
return (parameter_name_node, parameter_rvalue)
def save_parameters(cls, parameters_dictionary, context):
'''Метод для сохранения значений параметров.'''
context.parent['__parameters__'].set_parameters(parameters_dictionary)
return ''
@contextfunction
def pkg(context, * args):
if args:
package_atom = args[0]
else:
package_atom = context.parent['__parameters__']['package']
return package_atom
class TemplateEngine:
def __init__(self, directory_path='/',
datavars_module=Variables(),
appends_set=set()):
TemplateParametersChecker._inspect_formats_package()
CalculateExtension._parameters_set =\
TemplateParametersChecker.available_parameters
CalculateExtension._datavars = datavars_module
self.available_formats = TemplateParametersChecker.available_formats
self.available_appends = set()
self._datavars_module = datavars_module
self._parameters_object = ParametersContainer()
self._template_text = ''
self.environment = Environment(loader=FileSystemLoader(directory_path),
extensions=[CalculateExtension])
self.environment.context_class = CalculateContext
def change_directory(self, directory_path):
'''Метод для смены директории в загрузчике.'''
self.environment.loader = FileSystemLoader(directory_path)
def process_template(self, template_path, env=set()):
'''Метод для обработки файла шаблона, расположенного по указанному
пути.'''
CalculateContext._env_set = env
template = self.environment.get_template(template_path)
self._parameters_object = ParametersContainer(parameters_dictionary={})
self._template_text = template.render(
__datavars__=self._datavars_module,
__parameters__=self._parameters_object
)
def process_template_from_string(self, string, template_type, env=set()):
'''Метод для обработки текста шаблона.'''
CalculateContext._env_set = env
template = self.environment.from_string(string)
self._parameters_object = ParametersContainer(parameters_dictionary={})
self._template_text = template.render(
__datavars__=self._datavars_module,
__parameters__=self._parameters_object,
__template_type__=template_type,
__DIR__=DIR, __FILE__=FILE
)
@property
def parameters(self):
return self._parameters_object.parameters
@property
def template_text(self):
text, self._template_text = self._template_text, ''
return text