From 23b7799aefa19ac56183daab39d4387818e4df8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=81?= Date: Tue, 12 May 2020 18:13:36 +0300 Subject: [PATCH] The 'merge' parameter processing is implemented and debugged. The implementation of the new algorithm for template merging is started. --- calculate/templates/template_engine.py | 5 +- calculate/templates/template_processor.py | 119 +++++++++----------- run.py | 32 +++--- template_action_draft.py | 26 +++++ tests/templates/test_directory_processor.py | 2 +- tests/templates/test_template_engine.py | 10 -- 6 files changed, 99 insertions(+), 95 deletions(-) create mode 100644 template_action_draft.py diff --git a/calculate/templates/template_engine.py b/calculate/templates/template_engine.py index 5226e72..73d5a7b 100644 --- a/calculate/templates/template_engine.py +++ b/calculate/templates/template_engine.py @@ -81,8 +81,6 @@ class ParametersProcessor: inheritable_parameters = {'chmod', 'chown', 'autoupdate', 'env', 'package', 'action'} - to_check_while_parsing = {'package'} - available_appends = set() available_formats = set() @@ -266,7 +264,7 @@ class ParametersProcessor: if parameter_value and isinstance(parameter_value, str): return parameter_value else: - raise IncorrectParameter("'run' parameter value is nkt correct") + raise IncorrectParameter("'run' parameter value is not correct") def check_exec_parameter(self, parameter_value): if parameter_value and isinstance(parameter_value, str): @@ -483,7 +481,6 @@ class ParametersProcessor: def resolve_or_missing(context, key, missing=missing, env={}): '''Переопределение функции из для поиска значений переменных из jinja2. Ищет переменные в datavars.''' - print('THERE IS MY RESOLVE_OR_MISSING') datavars = context.parent['__datavars__'] if key in context.vars: diff --git a/calculate/templates/template_processor.py b/calculate/templates/template_processor.py index 010e598..bd41bbc 100644 --- a/calculate/templates/template_processor.py +++ b/calculate/templates/template_processor.py @@ -463,6 +463,15 @@ class DirectoryTree: directory_tree._tree = self._tree[directory] return directory_tree + def __getitem__(self, name): + if name in self._tree: + return self._tree[name] + else: + return None + + def __setitem__(self, name, value): + self._tree[name] = value + def __iter__(self): if self._tree is not None: return iter(self._tree.keys()) @@ -512,19 +521,23 @@ class DirectoryProcessor: main.cl_template_path.split(',')) self.inheritable_parameters = set( - ParametersProcessor.inheritable_parameters) + ParametersProcessor.inheritable_parameters) self.processed_packages = [] self.packages_to_merge = [] self.packages_file_trees = {} + def process_template_directories(self): + # Проходим каталоги из main.cl_template.path + # Режим заполнения очередей директорий пакетов, необходимых для более # быстрой обработки параметра merge. - self.fill_queues = True + if self.for_package: + self.fill_trees = True + else: + self.fill_trees = False - def process_template_directories(self): - # Проходим каталоги из main.cl_template.path for directory_path in self.template_paths: self.base_directory = directory_path.strip() entries = os.scandir(self.base_directory) @@ -535,10 +548,9 @@ class DirectoryProcessor: self.cl_chroot_path, ParametersContainer(), directory_tree=self.directory_tree) - print('\n#######################################\n') # Теперь когда дерево заполнено, можно выключить этот режим. - self.fill_queues = False + self.fill_trees = False if self.for_package: self.output.set_info('Processing packages from merge parameter...') @@ -547,13 +559,6 @@ class DirectoryProcessor: def merge_packages(self): not_merged_packages = [] - print('It was for package: {}'.format(self.for_package)) - print('Packages to merge: {}'.format(self.packages_to_merge)) - print('ALL TREES') - for package, tree in self.packages_file_trees.items(): - print('package: {}'.format(package)) - print('tree') - tree.show_tree() while self.packages_to_merge: self.for_package = self.packages_to_merge.pop() @@ -566,12 +571,9 @@ class DirectoryProcessor: not_merged_packages.append(self.for_package) continue - print('TREE FOR {}'.format(self.for_package)) - for directory_name in self.packages_file_trees[self.for_package]: directory_tree = self.packages_file_trees[self.for_package].\ get_directory_tree(directory_name) - print('current directory tree:') self.walk_directory_tree(directory_tree.base_directory, self.cl_chroot_path, @@ -579,6 +581,7 @@ class DirectoryProcessor: directory_tree=directory_tree) self.processed_packages.append(self.for_package) + if not_merged_packages: self.output.set_error('Packages {} is not merged.'. format(','.join(self.packages_to_merge))) @@ -586,7 +589,10 @@ class DirectoryProcessor: self.output.set_success('All packages are merged...') def get_directories_queue(self, path): + '''Уже не актуальный метод для построение очередей из путей к + шаблонам.''' directories_queue = [] + for base_dir in self.template_paths: base_dir = base_dir.strip() if path.startswith(base_dir): @@ -606,9 +612,8 @@ class DirectoryProcessor: directory_name = os.path.basename(current_directory_path) - if self.fill_queues: + if self.fill_trees: directory_tree[directory_name] = {} - pprint(directory_tree) current_target_path = target_path @@ -627,8 +632,8 @@ class DirectoryProcessor: # обрабатываем в первую очередь шаблон директории. if '.calculate_directory' in template_files: - print('.calculate_directory is found.') template_files.remove('.calculate_directory') + try: self.template_engine.process_template( '.calculate_directory', @@ -659,40 +664,33 @@ class DirectoryProcessor: current_target_path = join_paths(self.cl_chroot_path, directory_parameters.path) - if self.check_package_and_action( + if self.fill_trees and self.check_package_and_action( directory_parameters, current_directory_path, DIR, directory_tree=directory_tree): current_target_path = os.path.join(current_target_path, directory_name) + elif self.fill_trees and self.check_package_and_action( + directory_parameters, + current_directory_path, + DIR): + current_target_path = os.path.join(current_target_path, + directory_name) else: # Обновляем дерево директорий для данного пакета. - if (directory_tree[directory_name] is None and - self.fill_queues): - print('\ndirectory_tree: {}'.format( - self.directory_tree)) - print('packages_file_trees: {}\n'.format( - self.packages_file_trees)) + if (self.fill_trees and + directory_tree[directory_name] is None): package_name = directory_parameters.package if package_name in self.packages_file_trees: - print('Package is in package file trees') self.packages_file_trees[package_name].update_tree( copy.deepcopy(self.directory_tree) ) else: - print('Package is not in package file trees') directory_tree = DirectoryTree(self.base_directory) directory_tree.update_tree( copy.deepcopy(self.directory_tree)) self.packages_file_trees[package_name] = directory_tree - print( - 'Created in: {}'.format(self.packages_file_trees)) - self.output.set_error( - 'add directory {} to merge'.format( - directory_name - ) - ) directory_tree = {} return @@ -700,7 +698,6 @@ class DirectoryProcessor: self.packages_to_merge.extend(directory_parameters.merge) else: - print('.calculate_directory is found.') # Если .calculate_directory отсутствует -- создаем директорию # используя унаследованные параметры и имя самой директории. if not self.check_package_and_action( @@ -710,26 +707,18 @@ class DirectoryProcessor: directory_tree=directory_tree): # Обновляем дерево директорий для данного пакета. if (directory_tree[directory_name] is None and - self.fill_queues): + self.fill_trees): package_name = directory_parameters.package - print('\ndirectory_tree: {}'.format(self.directory_tree)) - print('packages_file_trees: {}\n'.format( - self.packages_file_trees)) if package_name in self.packages_file_trees: - print('Package is in package file trees') self.packages_file_trees[package_name].update_tree( copy.deepcopy(self.directory_tree) ) else: - print('Package is not in package file trees') directory_tree = DirectoryTree(self.base_directory) directory_tree.update_tree( copy.deepcopy(self.directory_tree)) self.packages_file_trees[package_name] = directory_tree - print( - 'Created in: {}'.format(self.packages_file_trees)) - self.output.set_error('add directory {} to merge'.format( - directory_name)) + directory_tree = {} return @@ -751,8 +740,8 @@ class DirectoryProcessor: format(current_directory_path)) # Если режим заполнения очередей выключен -- используем имеющуюся - # очередь. - if not self.fill_queues and directory_tree: + # очередь для получения списков обрабатываемых файлов и директорий. + if not self.fill_trees and directory_tree: template_files = [] template_directories = [] @@ -789,14 +778,18 @@ class DirectoryProcessor: template_text = self.template_engine.template_text - if not self.check_package_and_action( + if self.fill_trees and not self.check_package_and_action( directory_parameters, template_path, FILE, directory_tree=directory_tree[directory_name]): - self.output.set_error('add template {} to merge'.format( - template_name)) continue + elif not self.fill_trees and not self.check_package_and_action( + directory_parameters, + template_path, FILE): + continue + + # Если есть параметр merge добавляем в список if self.for_package and directory_parameters.merge: self.packages_to_merge.extend(directory_parameters.merge) @@ -826,44 +819,44 @@ class DirectoryProcessor: format(template_path)) # Обновляем дерево директорий для данного пакета. - if (self.fill_queues and directory_tree[directory_name]): + if (self.fill_trees and directory_tree[directory_name]): package_name = directory_parameters.package if package_name in self.packages_file_trees: - print('Package is in package file trees') self.packages_file_trees[package_name].update_tree( copy.deepcopy(self.directory_tree)) else: - print('Package is not in package file trees') directory_tree = DirectoryTree(self.base_directory) directory_tree.update_tree(copy.deepcopy(self.directory_tree)) self.packages_file_trees[package_name] = directory_tree - print( - 'Created in: {}'.format(self.packages_file_trees)) directory_tree[directory_name] = {} # проходимся далее по директориям. for directory in template_directories: - if self.fill_queues: + if self.fill_trees: self.walk_directory_tree( directory, current_target_path, directory_parameters.get_inheritables(), directory_tree=directory_tree[directory_name]) directory_tree[directory_name] = {} else: + if isinstance(directory, DirectoryTree): + directory_path = directory.base_directory + else: + directory_path = directory + self.walk_directory_tree( - directory.base_directory, + directory_path, current_target_path, - directory_parameters.get_inheritables(), - directory_tree=directory) + directory_parameters.get_inheritables()) - if self.fill_queues: + if self.fill_trees: directory_tree = {} return def check_package_and_action(self, parameters, template_path, - template_type, directory_tree={}): + template_type, directory_tree=None): if parameters.append != 'skip': if not parameters.action: self.output.set_warning( @@ -888,7 +881,7 @@ class DirectoryProcessor: # return False elif parameters.package != self.for_package: - if self.fill_queues: + if directory_tree is not None: template_name = os.path.basename(template_path) directory_tree[template_name] = None diff --git a/run.py b/run.py index f03810a..2c77702 100644 --- a/run.py +++ b/run.py @@ -25,20 +25,18 @@ template_engine.process_template_from_string( template_engine.parameters.print_parameters_for_debug() -# template_engine.parameters.print_parameters_for_debug() -# -# print('\nSECOND TEMPLATE\n') -# template_engine.process_template_from_string( -# '''{% calculate package = 'dev-lang/python-2.7', action = 'update' -%} -# {% if pkg('category/name') < '3.6.9' -%} -# The package exists. -# {% else -%} -# The package does not exist. -# {% endif -%} -# {% if Version('12.3.4-r1') < '12.2.5' %} -# Version is correct. -# {% endif -%}''', -# DIR) -# -# template_engine.parameters.print_parameters_for_debug() -# print('\nTEMPLATE TEXT:\n{0}'.format(template_engine.template_text)) +print('\nSECOND TEMPLATE\n') +template_engine.process_template_from_string( + '''{% calculate package = 'dev-lang/python-2.7', action = 'update' -%} +{% if pkg('category/name') < '3.6.9' -%} +The package exists. +{% else -%} +The package does not exist. +{% endif -%} +{% if Version('12.3.4-r1') < '12.2.5' %} +Version is correct. +{% endif -%}''', + DIR) + +template_engine.parameters.print_parameters_for_debug() +print('\nTEMPLATE TEXT:\n{0}'.format(template_engine.template_text)) diff --git a/template_action_draft.py b/template_action_draft.py new file mode 100644 index 0000000..f6d2ec0 --- /dev/null +++ b/template_action_draft.py @@ -0,0 +1,26 @@ +from calculate.templates.template_engine import TemplateEngine, Variables,\ + ConditionFailed, DIR, FILE +from calculate.templates.template_processor import TemplateAction + +template_text = '''{% calculate append = 'join', path = '/etc/file.conf' %} +{% calculate package = "dev-lang/python" %} +{% calculate name = 'filename', format = 'samba' %} +[section one] +parameter_1 = value_1 +!parameter_2 = value_2 +''' + +APPENDS_SET = TemplateAction().available_appends + +template_engine = TemplateEngine(appends_set=APPENDS_SET) +template_engine.process_template_from_string(template_text, FILE) +template_parameters = template_engine.parameters +target_path = '/etc/dir/file.conf' + + +def use_template(target_path, parameters, chroot_path): + print('Template parameters:') + parameters.print_parameters_for_debug() + + +use_template(target_path, template_parameters) diff --git a/tests/templates/test_directory_processor.py b/tests/templates/test_directory_processor.py index 9af0a33..5d733a8 100644 --- a/tests/templates/test_directory_processor.py +++ b/tests/templates/test_directory_processor.py @@ -77,4 +77,4 @@ class TestDirectoryProcessor: # pytest.fail('Unexpected exception: {}'.format(str(error))) # print('time: {}'.format(time.time() - start_time)) - assert False + # assert False diff --git a/tests/templates/test_template_engine.py b/tests/templates/test_template_engine.py index 9517e12..0f73001 100644 --- a/tests/templates/test_template_engine.py +++ b/tests/templates/test_template_engine.py @@ -4,14 +4,6 @@ from calculate.templates.template_engine import TemplateEngine, Variables,\ from calculate.templates.template_processor import TemplateAction -PARAMETERS_SET = {'name', 'path', 'append', 'chmod', 'chown', - 'autoupdate', 'env', 'link', 'force', 'symbolic', - 'format', 'comment', 'protected', 'mirror', 'run', - 'exec', 'env', 'stretch', 'convert', 'dotall', - 'multiline', 'dconf', 'package', 'merge', - 'postmerge', 'action'} - - APPENDS_SET = TemplateAction().available_appends @@ -108,8 +100,6 @@ class TestTemplateEngine(): template_engine.process_template_from_string(input_template, FILE) output_parameters = template_engine.parameters - print('env set = {}'.format(output_parameters.env)) - print('env set type = {}'.format(type(output_parameters.env))) assert (output_parameters.name == 'filename' and output_parameters.path == '/etc/other_path' and output_parameters.env == {'other_vars', 'vars'})