# vim: fileencoding=utf-8 # from jinja2.ext import Extension from jinja2 import Environment, FileSystemLoader, TemplateSyntaxError, nodes,\ contextfunction from jinja2.utils import missing from jinja2.runtime import Context, Undefined from collections.abc import MutableMapping from collections import OrderedDict from importlib import import_module from pprint import pprint import copy import re import os from ..utils.package import PackageAtomParser, PackageAtomError, NOTEXIST,\ Version from ..utils.files import join_paths, check_directory_link, check_command,\ FilesError from calculate.variables.datavars import HashType, NamespaceNode,\ VariableNode, IniType, IntegerType,\ FloatType, ListType from calculate.variables.loader import Datavars # Типы шаблона: директория или файл. DIR, FILE, LINK = range(3) class IncorrectParameter(Exception): pass class DefaultParameterError(Exception): pass class SaveError(Exception): pass class ConditionFailed(TemplateSyntaxError): pass class Variables(MutableMapping): '''Класс-заглушка вместо модуля переменных для тестов.''' def __init__(self, *args, **kwargs): self.__attrs = dict(*args, **kwargs) def __next__(self): iterator = iter(self.__attrs) return next(iterator) def __getattribute__(self, name: str): if name == '_Variables__attrs': return super().__getattribute__(name) if name == 'available_packages': return super().__getattribute__(name) if name == 'variables': return self.__attrs try: return self.__attrs[name] except KeyError: raise AttributeError(name) @property def available_packages(self) -> set: packages = set(self.__attrs.keys()) packages.update({'custom'}) return packages def __getitem__(self, name: str): return self.__attrs[name] def __setitem__(self, name: str, value) -> None: self.__attrs[name] = value def __delitem__(self, name: str) -> None: del self.__attrs[name] def __iter__(self): return iter(self.__attrs) def __len__(self) -> int: return len(self.__attrs) def __repr__(self) -> str: return ''.format(self.__attrs) def __contains__(self, name: str) -> bool: return name in self.__attrs class ParametersProcessor: '''Класс для проверки и разбора параметров шаблона.''' available_parameters = {'name', 'path', 'append', 'chmod', 'chown', 'autoupdate', 'env', 'force', 'source', 'format', 'unbound', 'mirror', 'run', 'exec', 'env', 'package', 'merge', 'postmerge', 'action', 'rebuild', 'restart', 'stop', 'start', 'handler', 'notify', 'group'} inheritable_parameters = {'chmod', 'chown', 'autoupdate', 'env', 'package', 'action', 'handler', 'group'} # Параметры по умолчанию для файлов -- # будут заполняться из __datavars__ file_default_parameters = {} # Параметры по умолчанию для директорий -- # будут заполняться из __datavars__ directory_default_parameters = {} available_appends = set() available_formats = dict() format_is_inspected = False chmod_value_regular = re.compile( r'([r-][w-][x-])([r-][w-][x-])([r-][w-][x-])') def __init__(self, parameters_container=None, chroot_path='/', datavars_module=Variables(), for_package=None): self.chroot_path = chroot_path self.template_type = DIR self.datavars_module = datavars_module self._parameters_container = parameters_container self.package_atom_parser = PackageAtomParser(chroot_path=chroot_path) self._groups = {} try: groups = list(datavars_module.main.cl.groups.variables.keys()) for group in groups: if isinstance(datavars_module, (Datavars, NamespaceNode)): packages = datavars_module.main.cl.groups[group].get_value( ).get_table() else: packages = datavars_module.main.cl.groups[group] self._groups.update({group: packages}) except Exception: pass self._inspect_formats_package() self._for_package = None # Если добавляемый параметр нуждается в проверке -- добавляем сюда # метод для проверки. self.checkers_list = OrderedDict({ 'package': self.check_package_parameter, 'group': self.check_group_parameter, 'append': self.check_append_parameter, 'rebuild': self.check_rebuild_parameter, 'restart': self.check_restart_parameter, 'run': self.check_run_parameter, 'exec': self.check_exec_parameter, 'stop': self.check_stop_parameter, 'start': self.check_start_parameter, 'chown': self.check_chown_parameter, 'chmod': self.check_chmod_parameter, 'autoupdate': self.check_autoupdate_parameter, 'source': self.check_source_parameter, 'force': self.check_force_parameter, 'env': self.check_env_parameter, 'merge': self.check_merge_parameter, 'format': self.check_format_parameter, 'handler': self.check_handler_parameter, 'notify': self.check_notify_parameter }) # Если добавляемый параметр должен быть проверен после того, как # будет закончен парсинг всех других параметров -- добавляем сюда метод # для проверки. self.postparse_checkers_list = OrderedDict({ 'package': self.check_postparse_package, 'append': self.check_postparse_append, 'source': self.check_postparse_source, 'autoupdate': self.check_postparse_autoupdate, 'run': self.check_postparse_run, 'exec': self.check_postparse_exec, 'handler': self.check_postparse_handler }) # Если параметр является наследуемым только при некоторых условиях -- # указываем здесь эти условия. self.inherit_conditions = {'chmod': self.is_chmod_inheritable} def set_parameters_container(self, parameters_container): '''Метод для установки текущего контейнера параметров.''' self._parameters_container = parameters_container self._added_parameters = set() @property def for_package(self): return self._for_package @for_package.setter def for_package(self, package): self._for_package = package def __getattr__(self, parameter_name): if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter: '{}'". format(parameter_name)) elif parameter_name not in self._parameters_container: return False else: return self._parameters_container[parameter_name] def check_template_parameter(self, parameter_name, parameter_value, template_type, lineno): '''Метод, проверяющий указанный параметр.''' self.lineno = lineno self.template_type = template_type if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter '{0}'". format(parameter_name)) elif parameter_name in self.checkers_list: checked_value = self.checkers_list[parameter_name](parameter_value) else: checked_value = parameter_value # Способ пропустить параметр, если он корректен, но добавлять не нужно if checked_value is None: return self._added_parameters.add(parameter_name) if (parameter_name in self.inheritable_parameters and self.template_type == DIR): if parameter_name in self.inherit_conditions: if self.inherit_conditions[parameter_name]( parameter_value): self._parameters_container.set_inheritable( {parameter_name: checked_value}) return else: self._parameters_container.set_inheritable( {parameter_name: checked_value}) return self._parameters_container.set_parameter( {parameter_name: checked_value}) def check_postparse_parameters(self): '''Метод, запускающий проверку параметров после их разбора.''' for parameter, parameter_checker in\ self.postparse_checkers_list.items(): if parameter not in self._added_parameters: continue parameter_value = self._parameters_container[parameter] result = parameter_checker(parameter_value) if result is not None: self._parameters_container.change_parameter(parameter, result) def check_template_parameters(self, parameters, template_type, lineno): '''Метод, запускающий проверку указанных параметров.''' self.template_type = template_type self.lineno = lineno for parameter_name in parameters: if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter '{0}'". format(parameter_name)) elif parameter_name in self.checkers_list: checked_value = self.checkers_list[parameter_name]( parameters[parameter_name] ) else: checked_value = parameters[parameter_name] if (template_type == DIR and parameter_name in self.inheritable_parameters): if parameter_name in self.inherit_conditions: if self.inherit_conditions[parameter_name]( parameters[parameter_name]): self._parameters_container.set_inheritable( parameter_name=checked_value) continue else: self._parameters_container.set_inheritable( parameter_name=checked_value) continue self._parameters_container.set_parameter( parameter_name=checked_value) # Методы для проверки параметров во время разбора шаблона. def check_package_parameter(self, parameter_value): if not isinstance(parameter_value, str): raise IncorrectParameter("'package' parameter must have value of" " the 'str' type, not" f" {type(parameter_value)}") return parameter_value def check_group_parameter(self, parameter_value): if isinstance(parameter_value, str): result = [group.strip() for group in parameter_value.split(',')] elif isinstance(parameter_value, (list, tuple)): result = parameter_value if isinstance(parameter_value, list) else\ list(parameter_value) else: raise IncorrectParameter("'package' parameter must have value of" " the 'str', 'list' or 'tuple' type, not" f" {type(parameter_value)}") for group in result: if group != "install" and group not in self._groups: raise IncorrectParameter(f"'group' parameter value '{group}'" " is not available. Available values:" f" {', '.join(self._groups.keys())}") return result def check_append_parameter(self, parameter_value): if parameter_value not in self.available_appends: raise IncorrectParameter("Unacceptable value '{}' of parameter" " 'append'".format(parameter_value)) return parameter_value def check_merge_parameter(self, parameter_value): packages_list = [] packages_names = parameter_value.split(',') for package_name in packages_names: package_name = package_name.strip() atom_object = self.package_atom_parser.parse_package_parameter( package_name) packages_list.append(atom_object) return packages_list def check_rebuild_parameter(self, parameter_value): if isinstance(parameter_value, bool): raise IncorrectParameter("'rebuild' parameter value is not bool") elif 'package' not in self._parameters_container: raise IncorrectParameter(("'rebuild' parameter is set without " "'package' parameter")) return parameter_value def check_restart_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter( "'restart' parameter value is not correct") def check_stop_parameter(self, parameter_value): if not parameter_value and isinstance(parameter_value, bool): raise IncorrectParameter("'stop' parameter value is empty") return parameter_value def check_start_parameter(self, parameter_value): if not parameter_value and isinstance(parameter_value, bool): raise IncorrectParameter("'start' parameter value is empty") return parameter_value def check_run_parameter(self, parameter_value): if self.template_type == DIR: raise IncorrectParameter("'run' parameter is not available in" " directory templates") if not parameter_value and isinstance(parameter_value, bool): raise IncorrectParameter("'run' parameter value is empty") try: interpreter_path = check_command(parameter_value) except FilesError: raise IncorrectParameter("interpreter from 'run' parameter not" " found") return interpreter_path def check_exec_parameter(self, parameter_value): if self.template_type == DIR: raise IncorrectParameter("'exec' parameter is not available in" " directory templates") if not parameter_value and isinstance(parameter_value, bool): raise IncorrectParameter("'exec' parameter value is empty") try: interpreter_path = check_command(parameter_value) except FilesError: raise IncorrectParameter("interpreter from 'exec' parameter is not" " found") return interpreter_path def check_chown_parameter(self, parameter_value): if not parameter_value or isinstance(parameter_value, bool): raise IncorrectParameter("'chown' parameter value is empty.") parameter_value = self.get_chown_values(parameter_value) return parameter_value def check_chmod_parameter(self, parameter_value): result = self.chmod_value_regular.search(parameter_value) if result: parameter_value = '' for group_number in range(3): current_group = result.groups()[group_number] num = '' for sym_number in range(3): if current_group[sym_number] != '-': num = num + '1' else: num = num + '0' parameter_value = parameter_value + num return int(parameter_value, 2) elif parameter_value.isdigit(): parameter_value = int(parameter_value, 8) return parameter_value else: raise IncorrectParameter("'chmod' parameter value is not correct") def check_source_parameter(self, parameter_value): if not parameter_value or isinstance(parameter_value, bool): raise IncorrectParameter("'source' parameter value is empty") if self.chroot_path != '/': real_path = join_paths(self.chroot_path, parameter_value) else: real_path = parameter_value # Ставим True, чтобы потом проверить этот параметр в postparse if not os.path.exists(real_path): return True source_file_type = DIR if os.path.isdir(real_path) else FILE # Проверяем, совпадают ли типы шаблона и файла, указанного в source if (self.template_type != source_file_type): raise IncorrectParameter( "the type of the 'source' file does not match" " the type of the template file") # Проверяем, не является ли файл из source зацикленной ссылкой. if (source_file_type == DIR and os.path.islink(real_path)): try: check_directory_link(real_path, chroot_path=self.chroot_path) except FilesError as error: raise IncorrectParameter( "the link from 'source' is not correct: {}". format(str(error))) return os.path.normpath(real_path) def check_env_parameter(self, parameter_value): env_set = set() for env_value in parameter_value.split(','): env_value = env_value.strip() if env_value not in self.datavars_module: raise ConditionFailed( "Modules from 'env' parameter do not exist.", self.lineno) else: env_set.add(env_value) # Если шаблон файла -- не добавляем env в контейнер, # а только используем для рендеринга шаблона. if self.template_type is None: return None if self._parameters_container.env: env_set.union(self._parameters_container.env) return env_set def check_force_parameter(self, parameter_value): if isinstance(parameter_value, bool): return parameter_value else: raise IncorrectParameter("'force' parameter value is not bool") def check_autoupdate_parameter(self, parameter_value): if isinstance(parameter_value, bool): return parameter_value else: raise IncorrectParameter( "'autoupdate' parameter value is not bool.") def check_format_parameter(self, parameter_value): if self.template_type == DIR: raise IncorrectParameter("'format' parameter is redundant for" " directory templates.") if isinstance(parameter_value, str): if parameter_value not in self.available_formats: raise IncorrectParameter(f"'{parameter_value}' value of the" " 'format' parameter is not" " available.") return parameter_value raise IncorrectParameter("'format' parameter must be string value not" f" {type(parameter_value)}.") def check_handler_parameter(self, parameter_value): if not isinstance(parameter_value, str): raise IncorrectParameter("'handler' parameter must be string" f" value not {type(parameter_value)}.") return parameter_value def check_notify_parameter(self, parameter_value): if isinstance(parameter_value, list): return parameter_value elif isinstance(parameter_value, str): return [parameter.strip() for parameter in parameter_value.split(',')] raise IncorrectParameter("'notify' parameter must be string or list" f" value not {type(parameter_value)}.") # Методы для проверки параметров после разбора всего шаблона. def check_postparse_append(self, parameter_value): if parameter_value == 'link': if 'source' not in self._parameters_container: raise IncorrectParameter("append = 'link' without source " "parameter.") if self._parameters_container.run: raise IncorrectParameter("'append' parameter is not 'compatible' " "with the 'run' parameter") if self._parameters_container.exec: raise IncorrectParameter("'append' parameter is not 'compatible' " "with the 'exec' parameter") def check_postparse_run(self, parameter_value): if self._parameters_container.append: raise IncorrectParameter("'run' parameter is not 'compatible' " "with the 'append' parameter") if self._parameters_container.exec: raise IncorrectParameter("'run' parameter is not 'compatible' " "with the 'exec' parameter") def check_postparse_exec(self, parameter_value): if self._parameters_container.append: raise IncorrectParameter("'exec' parameter is not 'compatible' " "with the 'append' parameter") if self._parameters_container.run: raise IncorrectParameter("'exec' parameter is not 'compatible' " "with the 'run' parameter") def check_postparse_source(self, parameter_value): # Если файл по пути source не существует, но присутствует параметр # mirror -- пропускаем шаблон для того, чтобы целевой файл мог быть # удален в исполнительном модуле. if parameter_value is True: if not self._parameters_container.mirror: raise IncorrectParameter( "File from 'source' parameter does not exist") elif (self.template_type == DIR and ('append' not in self._parameters_container or self._parameters_container['append'] != 'link')): raise IncorrectParameter( ("'source' parameter is set without " "append = 'link' for directory template") ) def check_postparse_autoupdate(self, parameter_value): if self._parameters_container.unbound: raise IncorrectParameter("'unbound' parameter is incompatible" " with 'autoupdate' parameter") def check_postparse_handler(self, parameter_value): if self._parameters_container.merge: raise IncorrectParameter("'merge' parameter is not available" " in handler templates.") elif (self._parameters_container.package and not self._parameters_container.is_inherited('package')): raise IncorrectParameter("'package' parameter is not available" " in handler templates.") def check_postparse_package(self, parameter_value): groups = [] package_atom = PackageAtomParser.parse_atom_name(parameter_value) if (self._parameters_container is None or not self._parameters_container.group): # Если параметр group не задан или метод используется для проверки # отдельного параметра package -- делаем только проверку install. # Предполагающую проверку существования пакета. groups.append('install') else: groups = self._parameters_container.group for group in groups: if group == 'install': try: result = self.package_atom_parser.parse_package_parameter( package_atom) return result except PackageAtomError as error: if error.errno != NOTEXIST: raise IncorrectParameter(error.message) elif self._check_package_group(package_atom, self._groups[group]): self._parameters_container.remove_parameter('package') return raise ConditionFailed(f"package '{parameter_value}'" " does not match the template condition", self.lineno) def _check_package_group(self, package: dict, group_packages: list): '''Метод для проверки соответствия описания пакета, заданного словарем, какому-либо описанию пакета, заданного в переменных groups.''' for group_package in group_packages: for parameter in ['category', 'name', 'version', 'slot']: if package[parameter] is not None: if (group_package[parameter] is None or group_package[parameter] != package[parameter]): break else: if package['use_flags'] is not None: if group_package['use_flags'] is None: continue else: for use_flag in package['use_flags']: if use_flag not in group_package['use_flags']: continue return True return False # Методы для проверки того, являются ли параметры наследуемыми. def is_chmod_inheritable(self, parameter_value): chmod_regex = re.compile(r'\d+') if chmod_regex.search(parameter_value): return False return True def get_chown_values(self, chown: str): """Получить значения uid и gid из параметра chown.""" if chown and ':' in chown: user_name, group_name = chown.split(':') if user_name.isdigit(): uid = int(user_name) else: import pwd try: if self.chroot_path == '/': uid = pwd.getpwnam(user_name).pw_uid else: uid = self.get_uid_from_passwd(user_name) except (FilesError, KeyError, TypeError) as error: raise IncorrectParameter( "'chown = {}' parameter check is failed: {}". format(chown, str(error))) if group_name.isdigit(): gid = int(group_name) else: import grp try: if self.chroot_path == '/': gid = grp.getgrnam(group_name).gr_gid else: gid = self.get_gid_from_group(group_name) except (FilesError, KeyError, TypeError) as error: raise IncorrectParameter( "'chown = {}' parameter check is failed: {}". format(chown, str(error))) return {'uid': uid, 'gid': gid} else: raise IncorrectParameter("'chown' value '{0}' is not correct". format(chown)) def get_uid_from_passwd(self, user_name: str): """Функция для получения uid из chroot passwd файла.""" passwd_file_path = os.path.join(self.chroot_path, 'etc/passwd') passwd_dictionary = dict() if os.path.exists(passwd_file_path): with open(passwd_file_path, 'r') as passwd_file: for line in passwd_file: if line.startswith('#'): continue passwd_item = tuple(line.split(':')[0:3:2]) if (len(passwd_item) > 1 and passwd_item[0]): passwd_dictionary[passwd_item[0]] = passwd_item[1] if user_name in passwd_dictionary: return int(passwd_dictionary[user_name]) else: raise FilesError("'{0}' uid was not found in {1}". format(user_name, passwd_file_path)) else: raise FilesError("passwd file was not found in {}". format(passwd_file_path)) def get_gid_from_group(self, group_name: str): """Функция для получения gid из chroot group файла.""" group_file_path = os.path.join(self.chroot_path, 'etc/group') group_dictionary = dict() if os.path.exists(group_file_path): with open(group_file_path, 'r') as group_file: for line in group_file: if line.startswith('#'): continue group_item = tuple(line.split(':')[0:3:2]) if len(group_item) > 1 and group_item[0] and group_item[1]: group_dictionary[group_item[0]] = group_item[1] if group_name in group_dictionary: return int(group_dictionary[group_name]) else: raise FilesError("'{0}' gid was not found in {1}". format(group_name, group_file_path)) else: raise FilesError("group file was not found in {}". format(group_file_path)) @classmethod def _inspect_formats_package(cls): '''Метод для определения множества доступных форматов и предоставляемых ими параметров.''' if cls.format_is_inspected: return parameters_set = set() available_formats = dict() format_directory_path = os.path.join(os.path.dirname(__file__), 'format') for module_name in os.listdir(format_directory_path): if (os.path.isdir(os.path.join(format_directory_path, module_name)) or module_name == '__init__.py'): continue if module_name.endswith('.py'): module_name = module_name[:-3] try: module = import_module('calculate.templates.format.{}'. format(module_name)) for obj in dir(module): if obj.endswith('Format') and obj != 'BaseFormat': format_class = getattr(module, obj, False) if format_class: format_name = getattr(format_class, 'FORMAT', False) if not format_name: continue available_formats.update( {format_name: format_class}) format_parameters = getattr(format_class, 'FORMAT_PARAMETERS', set()) parameters_set.update(format_parameters) except Exception: continue cls.available_formats = available_formats cls.available_parameters.update(parameters_set) cls.formats_inspected = True def resolve_or_missing(context, key, missing=missing, env={}): '''Переопределение функции из для поиска значений переменных из jinja2. Ищет переменные в datavars.''' datavars = context.parent['__datavars__'] if key in context.vars: return context.vars[key] if key in context.parent: return context.parent[key] if key in datavars: return datavars[key] for name in env: if name in datavars and key in datavars[name]: return datavars[name][key] return missing class CalculateContext(Context): '''Класс контекста позволяющий использовать значения datavars и сохранять их.''' _env_set = set() def resolve(self, key): if self._legacy_resolve_mode: rv = resolve_or_missing(self, key, env=self._env_set) else: rv = self.resolve_or_missing(key) if rv is missing: return self.environment.undefined(name=key) return rv def resolve_or_missing(self, key): if self._legacy_resolve_mode: rv = self.resolve(key) if isinstance(rv, Undefined): rv = missing return rv return resolve_or_missing(self, key, env=self._env_set) class ParametersContainer(MutableMapping): '''Класс для хранения параметров, взятых из шаблона, и передачи их шаблонизатору.''' def __init__(self, parameters_dictionary=None): # Слой ненаследуемых параметров. self.__parameters = {} # Слой наследуемых параметров. if parameters_dictionary is not None: self.__inheritable = parameters_dictionary else: self.__inheritable = {} def set_parameter(self, item_to_add: dict): self.__parameters.update(item_to_add) def set_inheritable(self, item_to_add: dict): self.__inheritable.update(item_to_add) def get_inheritables(self): return ParametersContainer(copy.deepcopy(self.__inheritable)) def remove_not_inheritable(self): self.__parameters.clear() def print_parameters_for_debug(self): print('Parameters:') pprint(self.__parameters) print('Inherited:') pprint(self.__inheritable) def is_inherited(self, parameter_name): return (parameter_name not in self.__parameters and parameter_name in self.__inheritable) def remove_parameter(self, parameter_name): if parameter_name in self.__parameters: self.__parameters.pop(parameter_name) elif parameter_name in self.__inheritable: self.__inheritable.pop(parameter_name) def change_parameter(self, parameter, value): if parameter in self.__parameters: self.__parameters.update({parameter: value}) elif parameter in self.__inheritable: self.__inheritable.update({parameter: value}) def _clear_container(self): self.__parameters.clear() self.__inheritable.clear() def __getattr__(self, parameter_name): if (parameter_name not in ParametersProcessor.available_parameters): raise IncorrectParameter("Unknown parameter: '{}'". format(parameter_name)) if parameter_name in self.__parameters: return self.__parameters[parameter_name] elif parameter_name in self.__inheritable: return self.__inheritable[parameter_name] else: return False def __getitem__(self, name): if name in self.__parameters: return self.__parameters[name] elif name in self.__inheritable: return self.__inheritable[name] else: return False def __setitem__(self, name, value): self.__parameters[name] = value def __delitem__(self, name): if name in self.__parameters: del self.__parameters[name] if name in self.__inheritable: del self.__inheritable[name] def __iter__(self): return iter(set(self.__parameters).union(self.__inheritable)) def __len__(self): return len(set(self.__parameters).union(self.__inheritable)) def __repr__(self): return ''.\ format(self.__parameters, self.__inheritable) def __contains__(self, name): return name in self.__parameters or name in self.__inheritable @property def parameters(self): return self.__parameters class CalculateExtension(Extension): '''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.''' _parameters_set = set() # Виды операций в теге save. ASSIGN, APPEND, REMOVE = range(3) def __init__(self, environment, parameters_processor: ParametersProcessor, datavars_module=Variables()): super().__init__(environment) self.environment = environment self.environment.globals.update({'pkg': self.pkg}) self._datavars = datavars_module self.parameters_processor = parameters_processor self.template_type = DIR # Флаг, указывающий, что тег calculate уже был разобран. Нужен для # того, чтобы проверять единственность тега calculate. self.calculate_parsed = False self.tags = {'calculate', 'save', 'set_var'} self.CONDITION_TOKENS_TYPES = {'eq', 'ne', 'lt', 'gt', 'lteq', 'gteq'} self.LITERAL_TOKENS_TYPES = {'string', 'integer', 'float'} if hasattr(self._datavars, 'variables_to_save'): self.TARGET_FILES_SET =\ set(self._datavars.variables_to_save.keys()) else: self.TARGET_FILES_SET = set() self.parse_methods = {'calculate': self.parse_calculate, 'save': self.parse_save} def __call__(self, env): # Необходимо для обеспечения возможности передать готовый объект # расширения, а не его класс. return self def parse(self, parser): self.parser = parser self.stream = parser.stream tag_token = self.stream.current.value return [self.parse_methods[tag_token]()] def parse_save(self): '''Метод для разбора тега save, сохраняющего значение указанной переменной datavars.''' lineno = next(self.stream).lineno # Целового файла больше не будет. target_file = nodes.Const('', lineno=lineno) if self.stream.skip_if('dot'): target_file_name = self.stream.expect('name').value if target_file_name in self.TARGET_FILES_SET: target_file = nodes.Const(target_file_name) else: raise TemplateSyntaxError("Unknown target file '{}'". format(target_file_name), lineno=lineno) # получаем список из имени переменной. module_name = self.stream.expect('name').value variable_name = [nodes.Const(module_name, lineno=lineno)] while self.stream.skip_if('dot'): name = self.stream.expect('name').value variable_name.append(nodes.Const(name, lineno=lineno)) variable_name = nodes.List(variable_name, lineno=lineno) if self.stream.skip_if('assign'): return self._make_save_node(variable_name, target_file, self.ASSIGN, lineno) elif self.stream.skip_if('add') and self.stream.skip_if('assign'): return self._make_save_node(variable_name, target_file, self.APPEND, lineno) elif self.stream.skip_if('sub') and self.stream.skip_if('assign'): return self._make_save_node(variable_name, target_file, self.REMOVE, lineno) raise TemplateSyntaxError("'=' is expected in 'save' tag", lineno=lineno) def parse_calculate(self): '''Метод для разбора тега calculate, содержащего значения параметров и условия выполнения шаблона.''' lineno = next(self.stream).lineno if self.calculate_parsed: raise TemplateSyntaxError( "template can only have one calculate tag.", lineno=lineno) expect_comma_flag = False conditions = [] while self.stream.current.type != 'block_end': if expect_comma_flag: self.stream.expect('comma') if (self.stream.current.type == 'name' and self.stream.current.value in self._parameters_set and self.stream.look().type != 'dot' and self.stream.look().type not in self.CONDITION_TOKENS_TYPES): # разбираем параметр. # pairs_list.append(self.get_parameter_node()) name_node, value_node = self._get_parameter() check_node = self.call_method('check_parameter', [name_node, value_node, nodes.ContextReference()], lineno=lineno) check_template_node = nodes.Template( [nodes.Output([check_node])]) check_template_node = check_template_node.set_environment( self.environment) check_template = self.environment.from_string( check_template_node) check_template.render(__datavars__=self._datavars) elif (self._is_variable_name(self.stream.current) or self.stream.current.type == 'lparen' or self.stream.current.type == 'integer'): # разбираем условие. Если условие False -- кидаем исключение. # condition_result = self.get_condition_result() # if not condition_result: # raise ConditionFailed('Condition is failed', # lineno=self.stream.current.lineno) conditions.append(self.parse_condition()) elif self.stream.current.type == 'name': raise TemplateSyntaxError( f"Unknown identifier '{self.stream.current.value}'" " in calculate tag.", lineno=self.stream.current.lineno) else: raise TemplateSyntaxError( f"Can not parse token '{self.stream.current.value}'" " in caluculate tag.", lineno=self.stream.current.lineno) expect_comma_flag = True self.parameters_processor.check_postparse_parameters() self.check_conditions(conditions) self.calculate_parsed = True return nodes.Output([nodes.Const('')], lineno=lineno) def _is_variable_name(self, token): if not token.type == 'name': return False if (token.value in self._datavars.available_packages or token.value in self.environment.globals): return True for env in self.environment.context_class._env_set: if token.value in self._datavars[env]: return True return False def check_parameter(self, parameter_name, parameter_value, context): self.parameters_processor.check_template_parameter( parameter_name, parameter_value, self.template_type, self.stream.current.lineno) return '' def parse_condition(self): try: condition_node = self.parser.parse_expression(with_condexpr=True) condition_node = self.call_method( 'set_condition_result', [condition_node], lineno=self.stream.current.lineno) condition_template = nodes.Template( [nodes.Output([condition_node])]) condition_template = condition_template.set_environment( self.environment) template = self.environment.from_string(condition_template) return template except Exception as error: raise ConditionFailed('Error during parsing condition:{}' .format(str(error)), lineno=self.stream.current.lineno) def check_conditions(self, conditions: list): for condition in conditions: self.condition_result = False try: condition.render(__datavars__=self._datavars) except Exception as error: raise ConditionFailed('Error during handling condition: {}' .format(str(error)), lineno=self.stream.current.lineno) if not self.condition_result: raise ConditionFailed('Condition is failed', lineno=self.stream.current.lineno) # DEPRECATED def get_condition_result(self): '''Метод для разбора условий из тега calculate.''' self.condition_result = False try: condition_node = self.parser.parse_expression(with_condexpr=True) condition_node = self.call_method( 'set_condition_result', [condition_node], lineno=self.stream.current.lineno) condition_template = nodes.Template( [nodes.Output([condition_node])]) condition_template = condition_template.set_environment( self.environment) template = self.environment.from_string(condition_template) template.render(__datavars__=self._datavars) except Exception: return False return self.condition_result def set_condition_result(self, condition_result): '''Метод для сохранения результата вычисления условия.''' self.condition_result = condition_result return '' def _make_save_node(self, variable_name_node, target_file_node, optype, lineno): '''Метод для создания ноды, сохраняющей переменные.''' right_value = self.parser.parse_expression(with_condexpr=True) optype_node = nodes.Const(optype, lineno=lineno) save_variable_node = self.call_method('save_variable', [variable_name_node, right_value, target_file_node, optype_node, nodes.ContextReference()], lineno=lineno) return nodes.Output([save_variable_node], lineno=lineno) def save_variable(self, variable, right_value, target_file, optype, context): '''Метод для сохранения значений переменных указанных в теге save.''' datavars = context.parent['__datavars__'] if variable[0] not in datavars: raise SaveError("can not save variable '{}'. The variable's" " package '{}' is not found".format( '.'.join(variable), variable[0])) modify_only = (variable[0] != 'custom') package = datavars[variable[0]] variable_name = variable[-1] value_container = self._find_value_container(variable, package, modify_only=modify_only) # Теперь меняем знaчение переменной. if isinstance(value_container, NamespaceNode): self._modify_variables(variable, value_container, right_value, optype, target=target_file, modify_only=modify_only) elif isinstance(value_container, VariableNode): hash_value = value_container.get_value().get_hash() if variable_name in hash_value: if optype == self.ASSIGN: new_value = right_value elif optype == self.APPEND: new_value = new_value + right_value elif optype == self.REMOVE: new_value = new_value - right_value else: new_value = right_value hash_value.update({variable_name: new_value}) value_container.set(hash_value) if target_file: self._save_to_target(variable[:-1], variable_name, new_value, target_file) return '' def _find_value_container(self, variable, vars_package, modify_only=True): '''Метод для поиска контейнера, путь к которому указан в аргументе. Этим контейнером может быть пространство имен или хэш.''' current_container = vars_package variable_path = variable[1:-1] for section in variable_path: if section in current_container.namespaces: current_container = current_container.namespaces[section] elif (section in current_container.variables and current_container. variables[section].variable_type is HashType): current_container = current_container.variables[section] if section != variable_path[-1]: # Если обнаружен хэш, но в пути к переменной кроме ключа # хэша есть еще что-то далее -- значит путь к переменной # ошибочен. raise SaveError("can not save variable '{}'. Other" " variable '{}' on the path".format( '.'.join(variable), current_container.get_fullname())) elif not modify_only: new_namespace = NamespaceNode(section) current_container.add_namespace(new_namespace) else: raise SaveError("can not save variable '{}'. Namespace '{}'" " is not found in '{}'".format( '.'.join(variable), section, current_container.get_fullname())) return current_container def _modify_variables(self, variable, namespace, new_value, optype, target='', modify_only=True): '''Метод для модификации значения переменной.''' variable_name = variable[-1] if variable_name in namespace.variables: variable_node = namespace[variable_name] if optype == self.ASSIGN: variable_node.set(new_value) elif optype == self.APPEND: new_value = self._append_variable_value(variable_node, new_value) variable_node.set(new_value) elif optype == self.REMOVE: new_value = self._remove_variable_value(variable_node, new_value) variable_node.set(new_value) elif not modify_only: VariableNode(variable_name, namespace, variable_type=IniType, source=str(new_value)) else: raise SaveError("can not create variable '{}' in not 'custom'" " namespace".format('.'.join(variable))) if target: if namespace.variables[variable_name].variable_type is HashType: for key, value in new_value.items(): self._save_to_target(variable, key, value, target) else: self._save_to_target(variable[:-1], variable_name, new_value, target) def _modify_hash(self, variable, hash_variable, new_value, optype, target=''): '''Метод для модификации значения в переменной-хэше.''' value_name = variable[-1] hash_value = hash_variable.get_value().get_hash() if value_name in hash_value: if optype == self.APPEND: new_value = hash_value[value_name] + new_value elif optype == self.REMOVE: new_value = hash_value[value_name] - new_value hash_value.update({value_name: new_value}) hash_variable.set(hash_value) if target: self._save_to_target(variable[:-1], value_name, new_value, target) def _save_to_target(self, namespace_name, variable_name, value, target): '''Метод для добавления переменной в список переменных, значение которых было установлено через тег save и при этом должно быть сохранено в указанном файле: save.target_file.''' namespace_name = tuple(namespace_name) target_file_dict = self._datavars.variables_to_save[target] if namespace_name not in target_file_dict: target_file_dict.update({namespace_name: dict()}) target_file_dict[namespace_name].update( {variable_name: ('=', str(value))}) def _append_variable_value(self, variable, value): '''Метод описывающий операцию += в теге save.''' variable_value = variable.get_value() if (variable.variable_type is IntegerType or variable.variable_type is FloatType): variable_value += value return variable_value elif variable.variable_type is ListType: if isinstance(value, str): value = value.split(',') if isinstance(value, list): for item in value: if item not in variable_value: variable_value.append(item) else: variable_value.append(value) return variable_value elif variable.variable_type is IniType: if not isinstance(variable_value, str): variable_value = str(variable_value) variable_value = variable_value.split(',') if not isinstance(value, list): if isinstance(value, str): value = value.split(',') else: value = str(value).split(',') for item in value: if item not in variable_value: variable_value.append(item) return ','.join(variable_value) # Пока что во всех остальных случаях будем просто возвращать исходное # значение. return variable_value def _remove_variable_value(self, variable, value): '''Метод описывающий операцию -= в теге save.''' variable_value = variable.get_value() if (variable.variable_type is IntegerType or variable.variable_type is FloatType): variable_value -= value return variable_value elif variable.variable_type is ListType: if isinstance(value, str): value = value.split(',') if isinstance(value, list): for item in value: if item in variable_value: variable_value.remove(item) elif value in variable_value: variable_value.remove(value) return variable_value elif variable.variable_type is IniType: if not isinstance(variable_value, list): if not isinstance(variable_value, str): variable_value = str(variable_value) variable_value = variable_value.split(',') if not isinstance(value, list): if isinstance(value, str): value = value.split(',') else: value = str(value).split(',') for item in value: if item in variable_value: variable_value.remove(item) return ','.join(variable_value) # Пока что во всех остальных случаях будем просто возвращать исходное # значение. return variable_value def _get_parameter(self): '''Метод для разбора параметров, содержащихся в теге calculate.''' lineno = self.stream.current.lineno parameter_name = self.stream.expect('name').value parameter_name_node = nodes.Const(parameter_name, lineno=lineno) if self.stream.skip_if('assign'): parameter_value = self.stream.current.value parameter_rvalue = self.parser.parse_expression(with_condexpr=True) if parameter_name == 'env': # если параметр env -- обновляем множество значений env # контекста вo время парсинга. env_names = parameter_value.split(',') for name in env_names: self.environment.context_class._env_set.add(name.strip()) else: parameter_rvalue = nodes.Const(True, lineno=lineno) parameter_value = True return (parameter_name_node, parameter_rvalue) def save_parameters(cls, parameters_dictionary, context): '''Метод для сохранения значений параметров.''' context.parent['__parameters__'].set_parameter(parameters_dictionary) return '' @contextfunction def pkg(self, context, *args) -> Version: '''Метод, реализующий функцию pkg() шаблонов. Функция предназначена для получения версии пакета, к которому уже привязан шаблон, если аргументов нет, или версию пакета в аргументе функции. Если аргументов нет, а шаблон не привязан к какому-либо пакету, или если указанного в аргументе пакета нет -- функция возвращает пустой объект Version().''' package_atom_parser = PackageAtomParser() if args: package_atom = args[0] try: atom_name = package_atom_parser.parse_package_parameter( package_atom) return atom_name.version except PackageAtomError: return Version() else: # package = context.parent['__parameters__'].package package = self.parameters_processor._parameters_container.package if not package: return Version() return package.version class TemplateEngine: def __init__(self, directory_path=None, datavars_module=Variables(), appends_set=set(), chroot_path='/', for_package=None): ParametersProcessor._inspect_formats_package() CalculateExtension._parameters_set =\ ParametersProcessor.available_parameters ParametersProcessor.available_appends = appends_set self._datavars_module = datavars_module self._template_text = '' self._parameters_object = ParametersContainer() self.parameters_processor = ParametersProcessor( chroot_path=chroot_path, datavars_module=datavars_module, for_package=for_package) if directory_path is not None: self.environment = Environment( loader=FileSystemLoader(directory_path)) else: self.environment = Environment() self.calculate_extension = CalculateExtension( self.environment, self.parameters_processor, datavars_module=datavars_module) self.environment.add_extension(self.calculate_extension) self.environment.context_class = CalculateContext @property def for_package(self): return self.parameters_processor.for_package @for_package.setter def for_package(self, package): self.parameters_processor.for_package = package def change_directory(self, directory_path): '''Метод для смены директории в загрузчике.''' self.environment.loader = FileSystemLoader(directory_path) def process_template(self, template_path, template_type, parameters=None): '''Метод для обработки файла шаблона, расположенного по указанному пути.''' if parameters is not None: self._parameters_object = parameters else: self._parameters_object = ParametersContainer() if self._parameters_object.env: CalculateContext._env_set = self._parameters_object.env.copy() else: CalculateContext._env_set = set() self.parameters_processor.set_parameters_container( self._parameters_object) self.calculate_extension.template_type = template_type self.calculate_extension.calculate_parsed = False template = self.environment.get_template(template_path) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, Version=Version ) def process_template_from_string(self, string, template_type, parameters=None): '''Метод для обработки текста шаблона.''' if parameters is not None: self._parameters_object = parameters else: self._parameters_object = ParametersContainer() if self._parameters_object.env: CalculateContext._env_set = self._parameters_object.env.copy() else: CalculateContext._env_set = set() self.parameters_processor.set_parameters_container( self._parameters_object) self.calculate_extension.template_type = template_type self.calculate_extension.calculate_parsed = False template = self.environment.from_string(string) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, Version=Version ) @property def parameters(self): return self._parameters_object @property def template_text(self): text, self._template_text = self._template_text, '' return text