# vim: fileencoding=utf-8 # from jinja2.ext import Extension from jinja2 import Environment, FileSystemLoader, TemplateSyntaxError, nodes,\ contextfunction from jinja2.utils import missing from jinja2.runtime import Context, Undefined from collections.abc import MutableMapping from collections import OrderedDict from importlib import import_module import re import os from ..utils.package import PackageAtom, PackageAtomError from ..utils.files import join_paths # Типы шаблона: директория или файл. DIR, FILE = range(2) class IncorrectParameter(Exception): pass class DefaultParameterError(Exception): pass class ConditionFailed(TemplateSyntaxError): pass class TemplateParametersChecker: '''Класс для хранения, проверки и разбора параметров шаблона.''' available_parameters = {'name', 'path', 'append', 'chmod', 'chown', 'autoupdate', 'env', 'force', 'source', 'format', 'protected', 'mirror', 'run', 'exec', 'env', 'package', 'merge', 'postmerge', 'action', 'rebuild', 'restart', 'stop', 'start'} inheritable_parameters = {'chmod': (None, None), 'chown': (None, None), 'autoupdate': (None, None), 'env': (None, None), 'package': (None, None), 'action': (None, None)} available_appends = set() directory_default_values = {'chown': 'root:root', 'chmod': '755'} file_default_values = {'chown': 'root:root', 'chmod': '644'} available_formats = set() format_is_inspected = False chmod_value_regular = re.compile( r'([r-][w-][x-])([r-][w-][x-])([r-][w-][x-])') package_atom_parser = PackageAtom() def __init__(self, parameters_container, template_type, parameters=dict(), chroot_path='/'): self.template_type = template_type self.chroot_path = chroot_path self._parameters_container = parameters_container self._inspect_formats_package() self.checkers_list = OrderedDict({ 'package': self.check_package_parameter, 'append': self.check_append_parameter, 'rebuild': self.check_rebuild_parameter, 'restart': self.check_restart_parameter, 'stop': self.check_stop_parameter, 'start': self.check_start_parameter, 'chown': self.check_chown_parameter, 'chmod': self.check_chmod_parameter, 'autoupdate': self.check_autoupdate_parameter, 'source': self.check_source_parameter, 'force': self.check_force_parameter }) try: if template_type == DIR: self.check_template_parameters(self.directory_default_values) elif template_type == FILE: self.check_template_parameters(self.file_default_values) except IncorrectParameter as error: raise DefaultParameterError('Default parameter value error: {}'. format(str(error))) if parameters: self.check_template_parameters(parameters) def __getattr__(self, parameter_name): if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter: '{}'". format(parameter_name)) elif parameter_name not in self._parameters_container: return False else: return self._parameters_container[parameter_name] def check_template_parameter(self, parameter_name, parameter_value): # Если параметр наследуем и уже встречался до этого с тем же # значением -- второй раз не проверяем его. if (parameter_name in self.inheritable_parameters and parameter_value == self.inheritable_parameters[parameter_name][0]): return self.inheritable_parameters[parameter_name][1] if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter '{0}'". format(parameter_name)) elif parameter_name in self.checkers_list: checked_value = self.checkers_list[parameter_name]( parameter_value) if parameter_name in self.inheritable_parameters: self.inheritable_parameters[parameter_name] =\ (parameter_value, checked_value) return checked_value def check_template_parameters(self, parameters): for parameter_name in parameters: if (parameter_name in self.inheritable_parameters and parameters[parameter_name] == self.inheritable_parameters[parameter_name][0]): self._parameters_container[parameter_name] =\ self.inheritable_parameters[parameter_name][1] continue if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter '{0}'". format(parameter_name)) elif parameter_name in self.checkers_list: parameter_value = self.checkers_list[parameter_name]( parameters[parameter_name] ) self._parameters_container[parameter_name] = parameter_value if parameter_name in self.inheritable_parameters: self.inheritable_parameters[parameter_name] =\ (parameter_value, parameters[parameter_name]) def check_package_parameter(self, parameter_value): try: self.package_atom_parser.parse_package_parameter(parameter_value) except PackageAtomError as error: raise IncorrectParameter(str(error)) parameter_value = self.package_atom_parser.atom_dictionary return parameter_value def check_append_parameter(self, parameter_value): if parameter_value not in self.available_appends: raise IncorrectParameter("Unacceptable value '{}' of parameter" " 'append'".format(parameter_value)) return parameter_value def check_rebuild_parameter(self, parameter_value): if isinstance(parameter_value, bool): raise IncorrectParameter("'rebuild' parameter value is not bool") elif 'package' not in self._parameters_container: raise IncorrectParameter(("'source' parameter is set without " "'package' parameter")) return parameter_value def check_restart_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter( "'restart' parameter value is not correct") def check_stop_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'stop' parameter value is not correct") def check_start_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'start' parameter value is not correct") def check_run_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'run' parameter value is nkt correct") def check_exec_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'exec' parameter value is not correct") def check_chown_parameter(self, parameter_value): if not parameter_value or isinstance(parameter_value, bool): raise IncorrectParameter("'chown' parameter value is empty.") parameter_value = self.get_chown_values(parameter_value) return parameter_value def check_chmod_parameter(self, parameter_value): result = self.chmod_value_regular.search(parameter_value) if result: parameter_value = '' for group_number in range(3): current_group = result.groups()[group_number] num = '' for sym_number in range(3): if current_group[sym_number] != '-': num = num + '1' else: num = num + '0' parameter_value = parameter_value + num return int(parameter_value, 2) elif parameter_value.isdigit(): parameter_value = int(parameter_value, 8) return parameter_value else: raise IncorrectParameter("'chmod' parameter value is not correct") def check_source_parameter(self, parameter_value): if self.chroot_path != '/': real_path = join_paths(self.chroot_path, parameter_value) else: real_path = parameter_value if not parameter_value or isinstance(parameter_value, bool): raise IncorrectParameter("'source' parameter value is empty") elif (self.template_type == DIR and ('append' not in self._parameters_container or self._parameters_container['append'] != 'link')): raise IncorrectParameter( ("'source' parameter is set without " "'append = link' for directory template") ) elif not os.path.exists(real_path): raise IncorrectParameter( "File from 'source' parameter does not exist") return os.path.normpath(real_path) def check_force_parameter(self, parameter_value): if isinstance(parameter_value, bool): return parameter_value else: raise IncorrectParameter("'force' parameter value is not bool") def check_autoupdate_parameter(self, parameter_value): print('autoupdate value = {}'.format(parameter_value)) if isinstance(parameter_value, bool): return parameter_value else: raise IncorrectParameter( "'autoupdate' parameter value is not bool") def get_chown_values(self, chown: str): """Получить значения uid и gid из параметра chown.""" if chown and ':' in chown: user_name, group_name = chown.split(':') if user_name.isdigit(): uid = int(user_name) else: import pwd try: if self.chroot_path == '/': uid = pwd.getpwnam(user_name).pw_uid else: uid = self.get_uid_from_passwd(user_name) except (KeyError, TypeError): self.output.set_error( format(user_name)) raise IncorrectParameter( ("'chown' value '{0}' is not correct:" "no such user in the system: {1}"). format(chown, user_name)) if group_name.isdigit(): gid = int(group_name) else: import grp try: if self.chroot_path == '/': gid = grp.getgrnam(group_name).gr_gid else: gid = self.get_gid_from_group(group_name) except (KeyError, TypeError): raise IncorrectParameter( ("'chown' value '{0}' is not correct:" "no such group in the system: {1}"). format(chown, group_name)) return {'uid': uid, 'gid': gid} else: raise IncorrectParameter("'chown' value '{0}' is not correct". format(chown, self.template_path)) def get_uid_from_passwd(self, user_name: str): """Взять uid из chroot passwd файла.""" passwd_file_path = os.path.join(self.chroot_path, 'etc/passwd') passwd_dictionary = [] if os.path.exists(passwd_file_path): with open(passwd_file_path, 'r') as passwd_file: for line in passwd_file: if line.startswith('#'): continue passwd_item = tuple(line.split(':')[0:3:2]) if (len(passwd_item) > 1 and passwd_item[0] and passwd_item[0]): passwd_dictionary.append(passwd_item) passwd_dictionary = dict(passwd_dictionary) return int(passwd_dictionary[user_name]) else: IncorrectParameter("passwd file was not found in {}". format(passwd_file_path)) def get_gid_from_group(self, group_name: str): """Взять gid из chroot group файла.""" group_file_path = os.path.join(self.chroot_path, 'etc/group') group_dictionary = [] if os.path.exists(group_file_path): with open(group_file_path, 'r') as group_file: for line in group_file: if line.startswith('#'): continue group_item = tuple(line.split(':')[0:3:2]) if len(group_item) > 1 and group_item[0] and group_item[1]: group_dictionary.append(group_item) group_dictionary = dict(group_dictionary) if group_name in group_dictionary: return int(group_dictionary[group_name]) else: IncorrectParameter("'{0}' gid was not found in {1}". format(group_name, group_file_path)) else: IncorrectParameter("group file was not found in {}". format(group_file_path)) @classmethod def _inspect_formats_package(cls): '''Метод для определения множества доступных форматов и предоставляемых ими параметров.''' if cls.format_is_inspected: return parameters_set = set() format_set = set() format_directory_path = os.path.join(os.path.dirname(__file__), 'format') for module_name in os.listdir(format_directory_path): if (os.path.isdir(os.path.join('format', module_name)) or module_name == '__init__.py'): continue if module_name.endswith('.py'): module_name = module_name[:-3] try: module = import_module('calculate.templates.format.{}'. format(module_name)) for obj in dir(module): if obj.endswith('Format') and obj != 'BaseFormat': format_class = getattr(module, obj, False) if format_class: format_set.add(format_class.FORMAT) parameters = getattr(format_class, 'FORMAT_PARAMETERS', set()) parameters_set.update(parameters) except Exception: continue cls.available_formats = format_set cls.available_parameters.update(parameters_set) cls.formats_inspected = True class Variables(MutableMapping): '''Класс заглушка вместо модуля переменных для тестов.''' def __init__(self, *args, **kwargs): self.__attrs = dict(*args, **kwargs) def __next__(self): iterator = iter(self.__attrs) return next(iterator) def __getattribute__(self, name): if name == '_Variables__attrs': return super().__getattribute__(name) try: return self.__attrs[name] except KeyError: raise AttributeError(name) def __getitem__(self, name): return self.__attrs[name] def __setitem__(self, name, value): self.__attrs[name] = value def __delitem__(self, name): del self.__attrs[name] def __iter__(self): return iter(self.__attrs) def __len__(self): return len(self.__attrs) def __repr__(self): return ''.format(self.__attrs) def resolve_or_missing(context, key, missing=missing, env={}): '''Переопределение функции из для поиска значений переменных из jinja2. Ищет переменные в datavars.''' datavars = context.parent['__datavars__'] if key in context.vars: return context.vars[key] if key in context.parent: return context.parent[key] if key in datavars: return datavars[key] for name in env: if name in datavars and key in datavars[name]: return datavars[name][key] return missing class CalculateContext(Context): '''Класс контекста позволяющий искать значения datavars и сохранять их.''' _env_set = set() def resolve(self, key): if self._legacy_resolve_mode: rv = resolve_or_missing(self, key, env=self._env_set) else: rv = self.resolve_or_missing(key) if rv is missing: return self.environment.undefined(name=key) return rv def resolve_or_missing(self, key): if self._legacy_resolve_mode: rv = self.resolve(key) if isinstance(rv, Undefined): rv = missing return rv return resolve_or_missing(self, key, env=self._env_set) class ParametersContainer(MutableMapping): '''Класс для хранения параметров, взятых из шаблона, и передачи их шаблонизатору.''' def __init__(self, parameters_dictionary={}): self.__parameters = parameters_dictionary self._new_template = True def set_parameters(self, *args, **kwargs): parameters = dict(*args, **kwargs) self.__parameters.update(parameters) def __getattr__(self, parameter_name): if (parameter_name not in TemplateParametersChecker.available_parameters): raise IncorrectParameter("Unknown parameter: '{}'". format(parameter_name)) elif parameter_name not in self.__parameters: return False else: return self.__parameters[parameter_name] def __getitem__(self, name): return self.__parameters[name] def __setitem__(self, name, value): self.__parameters[name] = value def __delitem__(self, name): del self.__parameters[name] def __iter__(self): return iter(self.__parameters) def __len__(self): return len(self.__parameters) def __repr__(self): return ''.format(self.__parameters) @property def parameters(self): return self.__parameters class CalculateExtension(Extension): '''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.''' _parameters_set = set() _datavars = Variables() def __init__(self, environment): print('EXTENSION IS INITIALIZED') self.tags = {'calculate', 'save', 'set_var'} self.CONDITION_TOKENS_TYPES = {'eq', 'ne', 'lt', 'gt', 'lteq', 'gteq'} self.LITERAL_TOKENS_TYPES = {'string', 'integer', 'float'} self.TARGET_FILES_SET = {'grp', 'system', 'etc', 'local', 'remote'} self.parse_methods = {'calculate': self.parse_calculate, 'save': self.parse_save} self.parameters_checker = None self.new_parameters_set self.environment = environment def parse(self, parser): self.parameters_checker = None self.parser = parser self.stream = parser.stream tag_token = self.stream.current.value return [self.parse_methods[tag_token]()] def parse_save(self): '''Метод для разбора тега save, сохраняющего значение указанной переменной datavars.''' lineno = next(self.stream).lineno target_file = nodes.Const('', lineno=lineno) if self.stream.skip_if('dot'): target_file_name = self.stream.expect('name').value if target_file_name in self.TARGET_FILES_SET: target_file = nodes.Const(target_file_name) else: TemplateSyntaxError("Unknown target file '{}'". format(target_file_name), lineno=lineno) # получаем список из имени переменной. module_name = self.stream.expect('name').value variable_name = [nodes.Const(module_name, lineno=lineno)] while self.stream.skip_if('dot'): name = self.stream.expect('name').value variable_name.append(nodes.Const(name, lineno=lineno)) variable_name = nodes.List(variable_name, lineno=lineno) if self.stream.skip_if('assign'): right_value = self.parser.parse_expression(with_condexpr=True) save_variable_node = self.call_method('save_variable', [variable_name, right_value, target_file, nodes.ContextReference()], lineno=lineno) return nodes.Output([save_variable_node], lineno=lineno) else: TemplateSyntaxError("'=' is expected in 'save' tag.", lineno=lineno) def parse_calculate(self): '''Метод для разбора тега calculate, содержащего значения параметров и условия выполнения шаблона.''' pairs_list = [] expect_comma_flag = False lineno = next(self.stream).lineno while self.stream.current.type != 'block_end': if expect_comma_flag: self.stream.expect('comma') if (self.stream.current.type == 'name' and self.stream.current.value in self._parameters_set and self.stream.look().type != 'dot' and self.stream.look().type not in self.CONDITION_TOKENS_TYPES): # разбираем параметр. # pairs_list.append(self.get_parameter_node()) name_node, value_node = self.get_parameter_node() check_node = self.call_method('check_parameter', [name_node, value_node, nodes.ContextReference()], lineno=lineno) pairs_list.append(check_node) elif (self.stream.current.type == 'name' or self.stream.current.type == 'lparen'): # разбираем условие. Если условие False -- кидаем исключение. condition_result = self.get_condition_result() if not condition_result: raise ConditionFailed( 'Condition is failed', lineno=self.stream.current.lineno ) else: raise TemplateSyntaxError('Name is expected in calculate tag.', lineno=self.stream.current.lineno) expect_comma_flag = True # dictionary_node = nodes.Dict(pairs_list) # save_node = self.call_method('save_parameters', # [dictionary_node, # nodes.ContextReference()], # lineno=lineno) # return nodes.Output([save_node], lineno=lineno) return nodes.Output(pairs_list, lineno=lineno) def check_parameter(self, parameter_name, parameter_value, context): parameters_object = context.parent['__parameters__'] if not self.parameters_checker: template_type = context.parent['__template_type__'] chroot_path = context.parent['__datavars__'].main.cl_chroot_path self.parameters_checker = TemplateParametersChecker( template_type, chroot_path=chroot_path ) parameters_object._new_template = True elif not parameters_object._new_template: print('pair = {0}: {1}'.format(parameter_name, parameter_value)) return '' def get_condition_result(self): '''Метод для разбора условий из тега calculate.''' condition_list = [] # собираем исходный код условия из токенов. # вероятно, следует придумать лучший способ. while (self.stream.current.type != 'block_end' and self.stream.current.type != 'comma'): if self.stream.current.type == 'string': condition_list.append("'{}'".format( self.stream.current.value )) elif self.stream.current.type == 'dot': self.stream.skip(1) if self.stream.current.type == 'name': next_name = '.' + self.stream.current.value else: raise TemplateSyntaxError( 'Variable name is not correct.', lineno=self.stream.current.lineno ) condition_list[-1] = condition_list[-1] + next_name else: condition_list.append( str(self.stream.current.value) ) self.stream.skip(1) condition = ' '.join(condition_list) # компилируем исходный код условия и получаем результат его вычисления. cond_expr = self.environment.compile_expression(condition) condition_result = cond_expr(__datavars__=self._datavars) return condition_result def save_variable(self, variable_name, right_value, target_file, context): '''Метод для сохранения значений переменных указанных в теге save.''' # временная реализация. datavars = context.parent['__datavars__'] module_name = variable_name[0] namespaces = variable_name[1:-1] variable_name = variable_name[-1] if module_name in datavars: variables_module = datavars[module_name] for namespace in namespaces: if namespace not in variables_module: variables_module[namespace] = Variables() variables_module = variables_module[namespace] variables_module[variable_name] = right_value else: AttributeError("Unknown variables module '{}'". format(module_name)) return '' def get_parameter_node(self): '''Метод для разбра параметров, содержащихся в теге calculate.''' lineno = self.stream.current.lineno parameter_name = self.stream.expect('name').value parameter_name_node = nodes.Const(parameter_name, lineno=lineno) if self.stream.skip_if('assign'): parameter_value = self.stream.current.value parameter_rvalue = self.parser.parse_expression(with_condexpr=True) if parameter_name == 'env': # если параметр env -- обновляем множенство значений env # контекста вo время парсинга. env_names = parameter_value.split(',') for name in env_names: CalculateContext._env_set.add(name.strip()) else: parameter_rvalue = nodes.Const(True, lineno=lineno) # return nodes.Pair(parameter_name_node, parameter_rvalue) return (parameter_name_node, parameter_rvalue) def save_parameters(cls, parameters_dictionary, context): '''Метод для сохранения значений параметров.''' context.parent['__parameters__'].set_parameters(parameters_dictionary) return '' @contextfunction def pkg(context, * args): if args: package_atom = args[0] else: package_atom = context.parent['__parameters__']['package'] return package_atom class TemplateEngine: def __init__(self, directory_path='/', datavars_module=Variables(), appends_set=set()): TemplateParametersChecker._inspect_formats_package() CalculateExtension._parameters_set =\ TemplateParametersChecker.available_parameters CalculateExtension._datavars = datavars_module self.available_formats = TemplateParametersChecker.available_formats self.available_appends = set() self._datavars_module = datavars_module self._parameters_object = ParametersContainer() self._template_text = '' self.environment = Environment(loader=FileSystemLoader(directory_path), extensions=[CalculateExtension]) self.environment.context_class = CalculateContext def change_directory(self, directory_path): '''Метод для смены директории в загрузчике.''' self.environment.loader = FileSystemLoader(directory_path) def process_template(self, template_path, env=set()): '''Метод для обработки файла шаблона, расположенного по указанному пути.''' CalculateContext._env_set = env template = self.environment.get_template(template_path) self._parameters_object = ParametersContainer(parameters_dictionary={}) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object ) def process_template_from_string(self, string, template_type, env=set()): '''Метод для обработки текста шаблона.''' CalculateContext._env_set = env template = self.environment.from_string(string) self._parameters_object = ParametersContainer(parameters_dictionary={}) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, __template_type__=template_type, __DIR__=DIR, __FILE__=FILE ) @property def parameters(self): return self._parameters_object.parameters @property def template_text(self): text, self._template_text = self._template_text, '' return text