From cc6dfc8de5ca71dbca22b843c33fec926c33d744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=81?= Date: Fri, 8 May 2020 18:50:56 +0300 Subject: [PATCH] pkg() function bug is fixed. The parameters processing and the conditions solving are in parsing time now --- calculate/templates/template_engine.py | 143 +++++++++++++++------- calculate/templates/template_processor.py | 15 ++- calculate/utils/package.py | 1 - run.py | 42 +++---- tests/templates/test_template_engine.py | 27 ++-- 5 files changed, 155 insertions(+), 73 deletions(-) diff --git a/calculate/templates/template_engine.py b/calculate/templates/template_engine.py index 39e85f6..5226e72 100644 --- a/calculate/templates/template_engine.py +++ b/calculate/templates/template_engine.py @@ -81,6 +81,8 @@ class ParametersProcessor: inheritable_parameters = {'chmod', 'chown', 'autoupdate', 'env', 'package', 'action'} + to_check_while_parsing = {'package'} + available_appends = set() available_formats = set() @@ -94,6 +96,8 @@ class ParametersProcessor: datavars_module=Variables()): self.chroot_path = chroot_path + self.template_type = DIR + self.datavars_module = datavars_module self._parameters_container = parameters_container @@ -479,6 +483,7 @@ class ParametersProcessor: def resolve_or_missing(context, key, missing=missing, env={}): '''Переопределение функции из для поиска значений переменных из jinja2. Ищет переменные в datavars.''' + print('THERE IS MY RESOLVE_OR_MISSING') datavars = context.parent['__datavars__'] if key in context.vars: @@ -601,15 +606,17 @@ class CalculateExtension(Extension): '''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.''' _parameters_set = set() - _datavars = Variables() - parameters_processor = None - def __init__(self, environment): + def __init__(self, environment, datavars_module=Variables()): super().__init__(environment) self.environment = environment + self.environment.globals.update({'pkg': self.pkg}) + self._datavars = datavars_module + self.template_type = DIR + self.tags = {'calculate', 'save', 'set_var'} self.CONDITION_TOKENS_TYPES = {'eq', 'ne', 'lt', 'gt', 'lteq', 'gteq'} self.LITERAL_TOKENS_TYPES = {'string', 'integer', 'float'} @@ -618,6 +625,9 @@ class CalculateExtension(Extension): self.parse_methods = {'calculate': self.parse_calculate, 'save': self.parse_save} + def __call__(self, env): + return self + def parse(self, parser): self.parser = parser self.stream = parser.stream @@ -663,7 +673,6 @@ class CalculateExtension(Extension): def parse_calculate(self): '''Метод для разбора тега calculate, содержащего значения параметров и условия выполнения шаблона.''' - pairs_list = [] expect_comma_flag = False lineno = next(self.stream).lineno @@ -680,12 +689,22 @@ class CalculateExtension(Extension): # разбираем параметр. # pairs_list.append(self.get_parameter_node()) name_node, value_node = self.get_parameter() + check_node = self.call_method('check_parameter', [name_node, value_node, nodes.ContextReference()], lineno=lineno) - pairs_list.append(check_node) + check_template_node = nodes.Template( + [nodes.Output([check_node])]) + + check_template_node = check_template_node.set_environment( + self.environment) + + check_template = self.environment.from_string( + check_template_node) + check_template.render(__datavars__=self._datavars) + elif (self.stream.current.type == 'name' or self.stream.current.type == 'lparen'): # разбираем условие. Если условие False -- кидаем исключение. @@ -707,49 +726,74 @@ class CalculateExtension(Extension): # lineno=lineno) # return nodes.Output([save_node], lineno=lineno) - return nodes.Output(pairs_list, lineno=lineno) + return nodes.Output([nodes.Const('')], lineno=lineno) def check_parameter(self, parameter_name, parameter_value, context): - template_type = context.parent['__template_type__'] - self.parameters_processor.check_template_parameter( parameter_name, parameter_value, - template_type, + self.template_type, self.stream.current.lineno) return '' def get_condition_result(self): '''Метод для разбора условий из тега calculate.''' - condition_list = [] + # лучший способ -- парсим в AST дерево, после чего компилируем и + # выполняем его. + self.condition_result = False + + try: + condition_node = self.parser.parse_expression(with_condexpr=True) + condition_node = self.call_method( + 'set_condition_result', + [condition_node], + lineno=self.stream.current.lineno) + condition_template = nodes.Template( + [nodes.Output([condition_node])]) + + condition_template = condition_template.set_environment( + self.environment) + template = self.environment.from_string(condition_template) + + template.render(__datavars__=self._datavars) + except Exception: + return False + + return self.condition_result + # собираем исходный код условия из токенов. # вероятно, следует придумать лучший способ. - while (self.stream.current.type != 'block_end' and - self.stream.current.type != 'comma'): - if self.stream.current.type == 'string': - condition_list.append("'{}'".format( - self.stream.current.value - )) - elif self.stream.current.type == 'dot': - self.stream.skip(1) - if self.stream.current.type == 'name': - next_name = '.' + self.stream.current.value - else: - raise TemplateSyntaxError( - 'Variable name is not correct.', - lineno=self.stream.current.lineno - ) - condition_list[-1] = condition_list[-1] + next_name - else: - condition_list.append(str(self.stream.current.value)) - self.stream.skip(1) - - condition = ' '.join(condition_list) + # while (self.stream.current.type != 'block_end' and + # self.stream.current.type != 'comma'): + # if self.stream.current.type == 'string': + # condition_list.append("'{}'".format( + # self.stream.current.value + # )) + # elif self.stream.current.type == 'dot': + # self.stream.skip(1) + # if self.stream.current.type == 'name': + # next_name = '.' + self.stream.current.value + # else: + # raise TemplateSyntaxError( + # 'Variable name is not correct.', + # lineno=self.stream.current.lineno + # ) + # condition_list[-1] = condition_list[-1] + next_name + # else: + # condition_list.append(str(self.stream.current.value)) + # self.stream.skip(1) + + # condition = ' '.join(condition_list) # компилируем исходный код условия и получаем результат его вычисления. - cond_expr = self.environment.compile_expression(condition) - condition_result = cond_expr(__datavars__=self._datavars) - return condition_result + + # cond_expr = self.environment.compile_expression(condition) + # condition_result = cond_expr(__datavars__=self._datavars) + # return condition_result + + def set_condition_result(self, condition_result): + self.condition_result = condition_result + return '' def save_variable(self, variable_name, right_value, target_file, context): '''Метод для сохранения значений переменных указанных в теге save.''' @@ -774,8 +818,10 @@ class CalculateExtension(Extension): def get_parameter(self): '''Метод для разбра параметров, содержащихся в теге calculate.''' lineno = self.stream.current.lineno + parameter_name = self.stream.expect('name').value parameter_name_node = nodes.Const(parameter_name, lineno=lineno) + if self.stream.skip_if('assign'): parameter_value = self.stream.current.value parameter_rvalue = self.parser.parse_expression(with_condexpr=True) @@ -787,7 +833,8 @@ class CalculateExtension(Extension): CalculateContext._env_set.add(name.strip()) else: parameter_rvalue = nodes.Const(True, lineno=lineno) - # return nodes.Pair(parameter_name_node, parameter_rvalue) + parameter_value = True + return (parameter_name_node, parameter_rvalue) def save_parameters(cls, parameters_dictionary, context): @@ -828,7 +875,9 @@ class TemplateEngine: self.available_formats = ParametersProcessor.available_formats self.available_appends = appends_set + ParametersProcessor.available_appends = appends_set + self._datavars_module = datavars_module self._template_text = '' @@ -839,8 +888,14 @@ class TemplateEngine: CalculateExtension.parameters_processor = self.parameters_processor - self.environment = Environment(loader=FileSystemLoader(directory_path), - extensions=[CalculateExtension]) + self.environment = Environment(loader=FileSystemLoader(directory_path)) + + self.calculate_extension = CalculateExtension( + self.environment, + datavars_module=datavars_module) + + self.environment.add_extension(self.calculate_extension) + self.environment.context_class = CalculateContext def change_directory(self, directory_path): @@ -853,6 +908,8 @@ class TemplateEngine: пути.''' if parameters is not None: self._parameters_object = parameters + else: + self._parameters_object = ParametersContainer() if self._parameters_object.env: CalculateContext._env_set = self._parameters_object.env.copy() @@ -862,12 +919,13 @@ class TemplateEngine: self.parameters_processor._parameters_container =\ self._parameters_object + self.calculate_extension.template_type = template_type + template = self.environment.get_template(template_path) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, - __template_type__=template_type, - __DIR__=DIR, __FILE__=FILE + Version=Version ) def process_template_from_string(self, string, template_type, @@ -875,6 +933,9 @@ class TemplateEngine: '''Метод для обработки текста шаблона.''' if parameters is not None: self._parameters_object = parameters + else: + self._parameters_object = ParametersContainer( + parameters_dictionary={}) if self._parameters_object.env: CalculateContext._env_set = self._parameters_object.env.copy() @@ -884,12 +945,12 @@ class TemplateEngine: self.parameters_processor._parameters_container =\ self._parameters_object + self.calculate_extension.template_type = template_type + template = self.environment.from_string(string) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object, - __template_type__=template_type, - __DIR__=DIR, __FILE__=FILE, Version=Version ) diff --git a/calculate/templates/template_processor.py b/calculate/templates/template_processor.py index 562f047..010e598 100644 --- a/calculate/templates/template_processor.py +++ b/calculate/templates/template_processor.py @@ -463,9 +463,18 @@ class DirectoryTree: directory_tree._tree = self._tree[directory] return directory_tree + def __iter__(self): + if self._tree is not None: + return iter(self._tree.keys()) + else: + return iter([]) + def __repr__(self): return ''.format(self._tree) + def __bool__(self): + return bool(self._tree) + class DirectoryProcessor: chmod_regex = re.compile(r'\d{3}') @@ -558,11 +567,11 @@ class DirectoryProcessor: continue print('TREE FOR {}'.format(self.for_package)) - self.packages_file_trees[self.for_package] for directory_name in self.packages_file_trees[self.for_package]: - directory_tree = self.packages_file_trees.get_directory_tree( - directory_name) + directory_tree = self.packages_file_trees[self.for_package].\ + get_directory_tree(directory_name) + print('current directory tree:') self.walk_directory_tree(directory_tree.base_directory, self.cl_chroot_path, diff --git a/calculate/utils/package.py b/calculate/utils/package.py index 8aea51c..27090f2 100644 --- a/calculate/utils/package.py +++ b/calculate/utils/package.py @@ -84,7 +84,6 @@ class Version: return version_value def _use_compare_operation(self, compare_operator, other_value): - '''Перегрузка x < y.''' version_value = self._version_value[:] other_value_length = len(other_value) diff --git a/run.py b/run.py index 0d900bb..f03810a 100644 --- a/run.py +++ b/run.py @@ -10,35 +10,35 @@ main = Variables({'cl_template_path': 'tests/templates/testfiles/template_dir_2'), 'cl_chroot_path': '/'}) +values = Variables({'val_1': 11, 'val_2': 13, 'val_3': 12}) -datavars = Variables({'main': main}) -version_object = Version('12.3.4-r1') -print('Version: {}'.format(version_object)) +datavars = Variables({'main': main, 'values': values}) template_engine = TemplateEngine( datavars_module=datavars, appends_set=TemplateAction().available_appends) template_engine.process_template_from_string( - ("{% calculate package = 'dev-lang/python', action = 'install'," - "chmod = 'rwxr-xr-x', append = 'join', force %}"), - DIR) + "{% calculate env = 'values', name = 'dir_name', append = 'join' %}", + DIR) template_engine.parameters.print_parameters_for_debug() -print('\nSECOND TEMPLATE\n') -template_engine.process_template_from_string( - '''{% calculate package = 'dev-lang/python-2.7', action = 'update' -%} -{% if pkg('category/name') < '3.6.9' -%} -The package exists. -{% else -%} -The package does not exist. -{% endif -%} -{% if Version('12.3.4-r1') < '12.2.5' %} -Version is correct. -{% endif -%}''', - DIR) - -template_engine.parameters.print_parameters_for_debug() -print('\nTEMPLATE TEXT:\n{0}'.format(template_engine.template_text)) +# template_engine.parameters.print_parameters_for_debug() +# +# print('\nSECOND TEMPLATE\n') +# template_engine.process_template_from_string( +# '''{% calculate package = 'dev-lang/python-2.7', action = 'update' -%} +# {% if pkg('category/name') < '3.6.9' -%} +# The package exists. +# {% else -%} +# The package does not exist. +# {% endif -%} +# {% if Version('12.3.4-r1') < '12.2.5' %} +# Version is correct. +# {% endif -%}''', +# DIR) +# +# template_engine.parameters.print_parameters_for_debug() +# print('\nTEMPLATE TEXT:\n{0}'.format(template_engine.template_text)) diff --git a/tests/templates/test_template_engine.py b/tests/templates/test_template_engine.py index c70d7f4..9517e12 100644 --- a/tests/templates/test_template_engine.py +++ b/tests/templates/test_template_engine.py @@ -42,7 +42,7 @@ class TestTemplateEngine(): output_parameters.autoupdate) def test_if_an_input_template_contains_env_parameter_in_which_module_name_is_assigned__the_variables_from_this_module_can_be_used_in_template_without_determining_of_their_module(self): - input_template = '''{% calculate name = vars.var_1, path = var_3, env = 'other_vars' %}''' + input_template = '''{% calculate name = vars.var_1, env = 'other_vars', path = var_3 %}''' datavars_module = Variables({'vars': Variables({'var_1': 'filename', 'var_2': '/etc/path'}), @@ -73,9 +73,11 @@ class TestTemplateEngine(): pytest.fail('Unexpected ConditionFailed exception.') def test_if_an_input_template_contains_several_conditions_and_it_is_False__the_template_engine_raises_ConditionFailed_exception(self): - input_template = '''{% calculate name = vars.var_1, var_4 < var_5 %} - {% calculate path = var_3, var_6 == 'value' %} - {% calculate env = 'other_vars'%}''' + input_template = '''{% calculate name = vars.var_1 %} + {% calculate env = 'other_vars'%} + {% calculate var_4 < var_5 %} + {% calculate path = var_3, var_6 == 'value' %}''' + datavars_module = Variables({'vars': Variables({'var_1': 'filename', 'var_2': '/etc/path'}), @@ -90,9 +92,10 @@ class TestTemplateEngine(): template_engine.process_template_from_string(input_template, DIR) def test_if_an_input_template_contains_several_calculate_tags__the_template_engine_will_parse_them_all_and_will_contain_all_parameters_and_result_of_all_conditions(self): - input_template = '''{% calculate name = vars.var_1, var_4 > var_5 %} - {% calculate path = var_3, var_6 == 'value' %} - {% calculate env = 'other_vars, vars'%}''' + input_template = '''{% calculate name = vars.var_1 %} + {% calculate env = 'other_vars, vars' %} + {% calculate var_4 > var_5 %} + {% calculate path = var_3, var_6 == 'value' %}''' datavars_module = Variables({'vars': Variables({'var_1': 'filename', 'var_2': '/etc/path'}), @@ -206,3 +209,13 @@ class TestTemplateEngine(): text = template_engine.template_text assert text == output_text + + def test_if_an_input_template_calculate_tag_contains_pkg_function_with_an_existing_package_in_its_argument__the_pkg_function_returns_version_value_of_the_package_from_package_parameter_without_any_exceptions(self): + input_template = '''{% calculate name = 'filename', package='dev-lang/python', force -%} + {% calculate pkg() > 2.7 -%}''' + + try: + template_engine = TemplateEngine(appends_set=APPENDS_SET) + template_engine.process_template_from_string(input_template, FILE) + except ConditionFailed: + pytest.fail('Unexpected ConditionFailed exception.')