# vim: fileencoding=utf-8 # from jinja2.ext import Extension from jinja2 import Environment, FileSystemLoader, TemplateSyntaxError, nodes,\ contextfunction from jinja2.utils import missing from jinja2.runtime import Context, Undefined from collections.abc import MutableMapping from collections import OrderedDict from importlib import import_module from pprint import pprint import copy import re import os from ..utils.package import PackageAtomParser, PackageAtomError, NOTEXIST,\ Version from ..utils.files import join_paths # Типы шаблона: директория или файл. DIR, FILE = range(2) class IncorrectParameter(Exception): pass class DefaultParameterError(Exception): pass class ConditionFailed(TemplateSyntaxError): pass class Variables(MutableMapping): '''Класс заглушка вместо модуля переменных для тестов.''' def __init__(self, *args, **kwargs): self.__attrs = dict(*args, **kwargs) def __next__(self): iterator = iter(self.__attrs) return next(iterator) def __getattribute__(self, name): if name == '_Variables__attrs': return super().__getattribute__(name) try: return self.__attrs[name] except KeyError: raise AttributeError(name) def __getitem__(self, name): return self.__attrs[name] def __setitem__(self, name, value): self.__attrs[name] = value def __delitem__(self, name): del self.__attrs[name] def __iter__(self): return iter(self.__attrs) def __len__(self): return len(self.__attrs) def __repr__(self): return ''.format(self.__attrs) class ParametersProcessor: '''Класс для проверки и разбора параметров шаблона.''' available_parameters = {'name', 'path', 'append', 'chmod', 'chown', 'autoupdate', 'env', 'force', 'source', 'format', 'protected', 'mirror', 'run', 'exec', 'env', 'package', 'merge', 'postmerge', 'action', 'rebuild', 'restart', 'stop', 'start'} inheritable_parameters = {'chmod', 'chown', 'autoupdate', 'env', 'package', 'action'} available_appends = set() available_formats = set() format_is_inspected = False chmod_value_regular = re.compile( r'([r-][w-][x-])([r-][w-][x-])([r-][w-][x-])') def __init__(self, parameters_container=None, chroot_path='/', datavars_module=Variables()): self.chroot_path = chroot_path self.template_type = DIR self.datavars_module = datavars_module self._parameters_container = parameters_container self.package_atom_parser = PackageAtomParser(chroot_path=chroot_path) self._inspect_formats_package() # Если добавляемый параметр нуждается в проверке -- добавляем сюда # метод для проверки. self.checkers_list = OrderedDict({ 'package': self.check_package_parameter, 'append': self.check_append_parameter, 'rebuild': self.check_rebuild_parameter, 'restart': self.check_restart_parameter, 'stop': self.check_stop_parameter, 'start': self.check_start_parameter, 'chown': self.check_chown_parameter, 'chmod': self.check_chmod_parameter, 'autoupdate': self.check_autoupdate_parameter, 'source': self.check_source_parameter, 'force': self.check_force_parameter, 'env': self.check_env_parameter, 'merge': self.check_merge_parameter }) # Если параметр является наследуемым только при некоторых условиях -- # указываем здесь эти условия. self.inherit_conditions = {'chmod': self.is_chmod_inheritable} def set_parameters_container(self, parameters_container): self._parameters_container = parameters_container def __getattr__(self, parameter_name): if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter: '{}'". format(parameter_name)) elif parameter_name not in self._parameters_container: return False else: return self._parameters_container[parameter_name] def check_template_parameter(self, parameter_name, parameter_value, template_type, lineno): self.lineno = lineno self.template_type = template_type if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter '{0}'". format(parameter_name)) elif parameter_name in self.checkers_list: checked_value = self.checkers_list[parameter_name](parameter_value) else: checked_value = parameter_value # Способ пропустить параметр, если он корректен, но добавлять не нужно if checked_value is None: return if (parameter_name in self.inheritable_parameters and self.template_type == DIR): if parameter_name in self.inherit_conditions: if self.inherit_conditions[parameter_name]( parameter_value): self._parameters_container.set_inheritable( {parameter_name: checked_value}) return else: self._parameters_container.set_inheritable( {parameter_name: checked_value}) return self._parameters_container.set_parameter( {parameter_name: checked_value}) def check_template_parameters(self, parameters, template_type, lineno): self.template_type = template_type self.lineno = lineno for parameter_name in parameters: if parameter_name not in self.available_parameters: raise IncorrectParameter("Unknown parameter '{0}'". format(parameter_name)) elif parameter_name in self.checkers_list: checked_value = self.checkers_list[parameter_name]( parameters[parameter_name] ) else: checked_value = parameters[parameter_name] if (template_type == DIR and parameter_name in self.inheritable_parameters): if parameter_name in self.inherit_conditions: if self.inherit_conditions[parameter_name]( parameters[parameter_name]): self._parameters_container.set_inheritable( parameter_name=checked_value) continue else: self._parameters_container.set_inheritable( parameter_name=checked_value) continue self._parameters_container.set_parameter( parameter_name=checked_value) def check_package_parameter(self, parameter_value): try: atom_object = self.package_atom_parser.parse_package_parameter( parameter_value) except PackageAtomError as error: if error.errno == NOTEXIST: raise ConditionFailed(error.message, self.lineno) else: raise IncorrectParameter(error.message) return atom_object def check_append_parameter(self, parameter_value): if parameter_value not in self.available_appends: raise IncorrectParameter("Unacceptable value '{}' of parameter" " 'append'".format(parameter_value)) return parameter_value def check_merge_parameter(self, parameter_value): packages_list = [] packages_names = parameter_value.split(',') for package_name in packages_names: package_name = package_name.strip() atom_object = self.package_atom_parser.parse_package_parameter( package_name) packages_list.append(atom_object) return packages_list def check_rebuild_parameter(self, parameter_value): if isinstance(parameter_value, bool): raise IncorrectParameter("'rebuild' parameter value is not bool") elif 'package' not in self._parameters_container: raise IncorrectParameter(("'source' parameter is set without " "'package' parameter")) return parameter_value def check_restart_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter( "'restart' parameter value is not correct") def check_stop_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'stop' parameter value is not correct") def check_start_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'start' parameter value is not correct") def check_run_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'run' parameter value is not correct") def check_exec_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): return parameter_value else: raise IncorrectParameter("'exec' parameter value is not correct") def check_chown_parameter(self, parameter_value): if not parameter_value or isinstance(parameter_value, bool): raise IncorrectParameter("'chown' parameter value is empty.") parameter_value = self.get_chown_values(parameter_value) return parameter_value def check_chmod_parameter(self, parameter_value): result = self.chmod_value_regular.search(parameter_value) if result: parameter_value = '' for group_number in range(3): current_group = result.groups()[group_number] num = '' for sym_number in range(3): if current_group[sym_number] != '-': num = num + '1' else: num = num + '0' parameter_value = parameter_value + num return int(parameter_value, 2) elif parameter_value.isdigit(): parameter_value = int(parameter_value, 8) return parameter_value else: raise IncorrectParameter("'chmod' parameter value is not correct") def check_source_parameter(self, parameter_value): if self.chroot_path != '/': real_path = join_paths(self.chroot_path, parameter_value) else: real_path = parameter_value if not parameter_value or isinstance(parameter_value, bool): raise IncorrectParameter("'source' parameter value is empty") elif (self.template_type == DIR and ('append' not in self._parameters_container or self._parameters_container['append'] != 'link')): raise IncorrectParameter( ("'source' parameter is set without " "'append = link' for directory template") ) elif not os.path.exists(real_path): raise IncorrectParameter( "File from 'source' parameter does not exist") return os.path.normpath(real_path) def check_env_parameter(self, parameter_value): env_set = set() for env_value in parameter_value.split(','): env_value = env_value.strip() if env_value not in self.datavars_module: raise ConditionFailed( "Modules from 'env' parameter do not exist.", self.lineno) else: env_set.add(env_value) # Если шаблон файла -- не добавляем env в контейнер, # а только используем для рендеринга шаблона. if self.template_type is None: return None if self._parameters_container.env: env_set.union(self._parameters_container.env) return env_set def check_force_parameter(self, parameter_value): if isinstance(parameter_value, bool): return parameter_value else: raise IncorrectParameter("'force' parameter value is not bool") def check_autoupdate_parameter(self, parameter_value): if isinstance(parameter_value, bool): return parameter_value else: raise IncorrectParameter( "'autoupdate' parameter value is not bool") def is_chmod_inheritable(self, parameter_value): chmod_regex = re.compile(r'\d+') if chmod_regex.search(parameter_value): return False return True def get_chown_values(self, chown: str): """Получить значения uid и gid из параметра chown.""" if chown and ':' in chown: user_name, group_name = chown.split(':') if user_name.isdigit(): uid = int(user_name) else: import pwd try: if self.chroot_path == '/': uid = pwd.getpwnam(user_name).pw_uid else: uid = self.get_uid_from_passwd(user_name) except (KeyError, TypeError): self.output.set_error( format(user_name)) raise IncorrectParameter( ("'chown' value '{0}' is not correct:" "no such user in the system: {1}"). format(chown, user_name)) if group_name.isdigit(): gid = int(group_name) else: import grp try: if self.chroot_path == '/': gid = grp.getgrnam(group_name).gr_gid else: gid = self.get_gid_from_group(group_name) except (KeyError, TypeError): raise IncorrectParameter( ("'chown' value '{0}' is not correct:" "no such group in the system: {1}"). format(chown, group_name)) return {'uid': uid, 'gid': gid} else: raise IncorrectParameter("'chown' value '{0}' is not correct". format(chown, self.template_path)) def get_uid_from_passwd(self, user_name: str): """Взять uid из chroot passwd файла.""" passwd_file_path = os.path.join(self.chroot_path, 'etc/passwd') passwd_dictionary = [] if os.path.exists(passwd_file_path): with open(passwd_file_path, 'r') as passwd_file: for line in passwd_file: if line.startswith('#'): continue passwd_item = tuple(line.split(':')[0:3:2]) if (len(passwd_item) > 1 and passwd_item[0] and passwd_item[0]): passwd_dictionary.append(passwd_item) passwd_dictionary = dict(passwd_dictionary) return int(passwd_dictionary[user_name]) else: IncorrectParameter("passwd file was not found in {}". format(passwd_file_path)) def get_gid_from_group(self, group_name: str): """Взять gid из chroot group файла.""" group_file_path = os.path.join(self.chroot_path, 'etc/group') group_dictionary = [] if os.path.exists(group_file_path): with open(group_file_path, 'r') as group_file: for line in group_file: if line.startswith('#'): continue group_item = tuple(line.split(':')[0:3:2]) if len(group_item) > 1 and group_item[0] and group_item[1]: group_dictionary.append(group_item) group_dictionary = dict(group_dictionary) if group_name in group_dictionary: return int(group_dictionary[group_name]) else: IncorrectParameter("'{0}' gid was not found in {1}". format(group_name, group_file_path)) else: IncorrectParameter("group file was not found in {}". format(group_file_path)) @classmethod def _inspect_formats_package(cls): '''Метод для определения множества доступных форматов и предоставляемых ими параметров.''' if cls.format_is_inspected: return parameters_set = set() format_set = set() format_directory_path = os.path.join(os.path.dirname(__file__), 'format') for module_name in os.listdir(format_directory_path): if (os.path.isdir(os.path.join('format', module_name)) or module_name == '__init__.py'): continue if module_name.endswith('.py'): module_name = module_name[:-3] try: module = import_module('calculate.templates.format.{}'. format(module_name)) for obj in dir(module): if obj.endswith('Format') and obj != 'BaseFormat': format_class = getattr(module, obj, False) if format_class: format_set.add(format_class.FORMAT) parameters = getattr(format_class, 'FORMAT_PARAMETERS', set()) parameters_set.update(parameters) except Exception: continue cls.available_formats = format_set cls.available_parameters.update(parameters_set) cls.formats_inspected = True def resolve_or_missing(context, key, missing=missing, env={}): '''Переопределение функции из для поиска значений переменных из jinja2. Ищет переменные в datavars.''' datavars = context.parent['__datavars__'] if key in context.vars: return context.vars[key] if key in context.parent: return context.parent[key] if key in datavars: return datavars[key] for name in env: if name in datavars and key in datavars[name]: return datavars[name][key] return missing class CalculateContext(Context): '''Класс контекста позволяющий использовать значения datavars и сохранять их.''' _env_set = set() def resolve(self, key): if self._legacy_resolve_mode: rv = resolve_or_missing(self, key, env=self._env_set) else: rv = self.resolve_or_missing(key) if rv is missing: return self.environment.undefined(name=key) return rv def resolve_or_missing(self, key): if self._legacy_resolve_mode: rv = self.resolve(key) if isinstance(rv, Undefined): rv = missing return rv return resolve_or_missing(self, key, env=self._env_set) class ParametersContainer(MutableMapping): '''Класс для хранения параметров, взятых из шаблона, и передачи их шаблонизатору.''' def __init__(self, parameters_dictionary={}): # Слой ненаследуемых параметров. self.__parameters = {} # Слой наследуемых параметров. self.__inheritable = parameters_dictionary def set_parameter(self, item_to_add: dict): self.__parameters.update(item_to_add) def set_inheritable(self, item_to_add: dict): self.__inheritable.update(item_to_add) def get_inheritables(self): return ParametersContainer(copy.deepcopy(self.__inheritable)) def remove_not_inheritable(self): self.__parameters.clear() def print_parameters_for_debug(self): print('Parameters:') pprint(self.__parameters) print('Inherited:') pprint(self.__inheritable) def __getattr__(self, parameter_name): if (parameter_name not in ParametersProcessor.available_parameters): raise IncorrectParameter("Unknown parameter: '{}'". format(parameter_name)) if parameter_name in self.__parameters: return self.__parameters[parameter_name] elif parameter_name in self.__inheritable: return self.__inheritable[parameter_name] else: return False def __getitem__(self, name): if name in self.__parameters: return self.__parameters[name] elif name in self.__inheritable: return self.__inheritable[name] else: raise KeyError() def __setitem__(self, name, value): self.__parameters[name] = value def __delitem__(self, name): if name in self.__parameters: del self.__parameters[name] if name in self.__inheritable: del self.__inheritable[name] def __iter__(self): return iter(set(self.__parameters).union(self.__inheritable)) def __len__(self): return len(set(self.__parameters).union(self.__inheritable)) def __repr__(self): return ''.\ format(self.__parameters, self.__inheritable) @property def parameters(self): return self.__parameters class CalculateExtension(Extension): '''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.''' _parameters_set = set() parameters_processor = None def __init__(self, environment, datavars_module=Variables()): super().__init__(environment) self.environment = environment self.environment.globals.update({'pkg': self.pkg}) self._datavars = datavars_module self.template_type = DIR self.tags = {'calculate', 'save', 'set_var'} self.CONDITION_TOKENS_TYPES = {'eq', 'ne', 'lt', 'gt', 'lteq', 'gteq'} self.LITERAL_TOKENS_TYPES = {'string', 'integer', 'float'} self.TARGET_FILES_SET = {'grp', 'system', 'etc', 'local', 'remote'} self.parse_methods = {'calculate': self.parse_calculate, 'save': self.parse_save} def __call__(self, env): return self def parse(self, parser): self.parser = parser self.stream = parser.stream tag_token = self.stream.current.value return [self.parse_methods[tag_token]()] def parse_save(self): '''Метод для разбора тега save, сохраняющего значение указанной переменной datavars.''' lineno = next(self.stream).lineno target_file = nodes.Const('', lineno=lineno) if self.stream.skip_if('dot'): target_file_name = self.stream.expect('name').value if target_file_name in self.TARGET_FILES_SET: target_file = nodes.Const(target_file_name) else: TemplateSyntaxError("Unknown target file '{}'". format(target_file_name), lineno=lineno) # получаем список из имени переменной. module_name = self.stream.expect('name').value variable_name = [nodes.Const(module_name, lineno=lineno)] while self.stream.skip_if('dot'): name = self.stream.expect('name').value variable_name.append(nodes.Const(name, lineno=lineno)) variable_name = nodes.List(variable_name, lineno=lineno) if self.stream.skip_if('assign'): right_value = self.parser.parse_expression(with_condexpr=True) save_variable_node = self.call_method('save_variable', [variable_name, right_value, target_file, nodes.ContextReference()], lineno=lineno) return nodes.Output([save_variable_node], lineno=lineno) else: TemplateSyntaxError("'=' is expected in 'save' tag.", lineno=lineno) def parse_calculate(self): '''Метод для разбора тега calculate, содержащего значения параметров и условия выполнения шаблона.''' expect_comma_flag = False lineno = next(self.stream).lineno while self.stream.current.type != 'block_end': if expect_comma_flag: self.stream.expect('comma') if (self.stream.current.type == 'name' and self.stream.current.value in self._parameters_set and self.stream.look().type != 'dot' and self.stream.look().type not in self.CONDITION_TOKENS_TYPES): # разбираем параметр. # pairs_list.append(self.get_parameter_node()) name_node, value_node = self.get_parameter() check_node = self.call_method('check_parameter', [name_node, value_node, nodes.ContextReference()], lineno=lineno) check_template_node = nodes.Template( [nodes.Output([check_node])]) check_template_node = check_template_node.set_environment( self.environment) check_template = self.environment.from_string( check_template_node) check_template.render(__datavars__=self._datavars) elif (self.stream.current.type == 'name' or self.stream.current.type == 'lparen'): # разбираем условие. Если условие False -- кидаем исключение. condition_result = self.get_condition_result() if not condition_result: raise ConditionFailed( 'Condition is failed', lineno=self.stream.current.lineno ) else: raise TemplateSyntaxError('Name is expected in calculate tag.', lineno=self.stream.current.lineno) expect_comma_flag = True # dictionary_node = nodes.Dict(pairs_list) # save_node = self.call_method('save_parameters', # [dictionary_node, # nodes.ContextReference()], # lineno=lineno) # return nodes.Output([save_node], lineno=lineno) return nodes.Output([nodes.Const('')], lineno=lineno) def check_parameter(self, parameter_name, parameter_value, context): self.parameters_processor.check_template_parameter( parameter_name, parameter_value, self.template_type, self.stream.current.lineno) return '' def get_condition_result(self): '''Метод для разбора условий из тега calculate.''' # лучший способ -- парсим в AST дерево, после чего компилируем и # выполняем его. self.condition_result = False try: condition_node = self.parser.parse_expression(with_condexpr=True) condition_node = self.call_method( 'set_condition_result', [condition_node], lineno=self.stream.current.lineno) condition_template = nodes.Template( [nodes.Output([condition_node])]) condition_template = condition_template.set_environment( self.environment) template = self.environment.from_string(condition_template) template.render(__datavars__=self._datavars) except Exception: return False return self.condition_result # собираем исходный код условия из токенов. # вероятно, следует придумать лучший способ. # while (self.stream.current.type != 'block_end' and # self.stream.current.type != 'comma'): # if self.stream.current.type == 'string': # condition_list.append("'{}'".format( # self.stream.current.value # )) # elif self.stream.current.type == 'dot': # self.stream.skip(1) # if self.stream.current.type == 'name': # next_name = '.' + self.stream.current.value # else: # raise TemplateSyntaxError( # 'Variable name is not correct.', # lineno=self.stream.current.lineno # ) # condition_list[-1] = condition_list[-1] + next_name # else: # condition_list.append(str(self.stream.current.value)) # self.stream.skip(1) # condition = ' '.join(condition_list) # компилируем исходный код условия и получаем результат его вычисления. # cond_expr = self.environment.compile_expression(condition) # condition_result = cond_expr(__datavars__=self._datavars) # return condition_result def set_condition_result(self, condition_result): self.condition_result = condition_result return '' def save_variable(self, variable_name, right_value, target_file, context): '''Метод для сохранения значений переменных указанных в теге save.''' # временная реализация. datavars = context.parent['__datavars__'] module_name = variable_name[0] namespaces = variable_name[1:-1] variable_name = variable_name[-1] if module_name in datavars: variables_module = datavars[module_name] for namespace in namespaces: if namespace not in variables_module: variables_module[namespace] = Variables() variables_module = variables_module[namespace] variables_module[variable_name] = right_value else: AttributeError("Unknown variables module '{}'". format(module_name)) return '' def get_parameter(self): '''Метод для разбра параметров, содержащихся в теге calculate.''' lineno = self.stream.current.lineno parameter_name = self.stream.expect('name').value parameter_name_node = nodes.Const(parameter_name, lineno=lineno) if self.stream.skip_if('assign'): parameter_value = self.stream.current.value parameter_rvalue = self.parser.parse_expression(with_condexpr=True) if parameter_name == 'env': # если параметр env -- обновляем множество значений env # контекста вo время парсинга. env_names = parameter_value.split(',') for name in env_names: CalculateContext._env_set.add(name.strip()) else: parameter_rvalue = nodes.Const(True, lineno=lineno) parameter_value = True return (parameter_name_node, parameter_rvalue) def save_parameters(cls, parameters_dictionary, context): '''Метод для сохранения значений параметров.''' context.parent['__parameters__'].set_parameter(parameters_dictionary) return '' @contextfunction def pkg(self, context, *args): package_atom_parser = PackageAtomParser() if args: package_atom = args[0] try: atom_name = package_atom_parser.parse_package_parameter( package_atom) return atom_name.version except PackageAtomError: return Version() else: # package = context.parent['__parameters__'].package package = self.parameters_processor._parameters_container.package if not package: return Version() return package.version class TemplateEngine: def __init__(self, directory_path='/', datavars_module=Variables(), appends_set=set(), chroot_path='/'): ParametersProcessor._inspect_formats_package() CalculateExtension._parameters_set =\ ParametersProcessor.available_parameters CalculateExtension._datavars = datavars_module self.available_formats = ParametersProcessor.available_formats self.available_appends = appends_set ParametersProcessor.available_appends = appends_set self._datavars_module = datavars_module self._template_text = '' self._parameters_object = ParametersContainer() self.parameters_processor = ParametersProcessor( chroot_path=chroot_path, datavars_module=datavars_module) CalculateExtension.parameters_processor = self.parameters_processor self.environment = Environment(loader=FileSystemLoader(directory_path)) self.calculate_extension = CalculateExtension( self.environment, datavars_module=datavars_module) self.environment.add_extension(self.calculate_extension) self.environment.context_class = CalculateContext def change_directory(self, directory_path): '''Метод для смены директории в загрузчике.''' self.environment.loader = FileSystemLoader(directory_path) def process_template(self, template_path, template_type, parameters=None): '''Метод для обработки файла шаблона, расположенного по указанному пути.''' if parameters is not None: self._parameters_object = parameters else: self._parameters_object = ParametersContainer() if self._parameters_object.env: CalculateContext._env_set = self._parameters_object.env.copy() else: CalculateContext._env_set = set() self.parameters_processor._parameters_container =\ self._parameters_object self.calculate_extension.template_type = template_type template = self.environment.get_template(template_path) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, Version=Version ) def process_template_from_string(self, string, template_type, parameters=None): '''Метод для обработки текста шаблона.''' if parameters is not None: self._parameters_object = parameters else: self._parameters_object = ParametersContainer( parameters_dictionary={}) if self._parameters_object.env: CalculateContext._env_set = self._parameters_object.env.copy() else: CalculateContext._env_set = set() self.parameters_processor._parameters_container =\ self._parameters_object self.calculate_extension.template_type = template_type template = self.environment.from_string(string) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, Version=Version ) @property def parameters(self): return self._parameters_object @property def template_text(self): text, self._template_text = self._template_text, '' return text