From 0b42d43438ddeeb99b4c6343e6f56f0767d8c611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=81?= Date: Thu, 8 Oct 2020 17:37:21 +0300 Subject: [PATCH] Execution sequence of packages from the merge parameter is changed. fixed 2 --- calculate/templates/template_processor.py | 66 ++++++++++++------- tests/templates/test_directory_processor.py | 32 ++++++++- .../templates_36/install/.calculate_directory | 1 + .../templates_36/install/script_0 | 14 ++++ .../templates_36/install/script_1 | 13 ++++ .../templates_36/install/script_2 | 12 ++++ 6 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_36/install/.calculate_directory create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_0 create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_1 create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_2 diff --git a/calculate/templates/template_processor.py b/calculate/templates/template_processor.py index dd99c6e..1ea8295 100644 --- a/calculate/templates/template_processor.py +++ b/calculate/templates/template_processor.py @@ -1689,13 +1689,13 @@ class DirectoryProcessor: self.template_paths = self.datavars_module.main.cl_template_path # Список обработанных пакетов. - self.processed_packages = [] + self.processed_packages = set() # Список пакетов, взятый из значений параметра merge. - self.packages_to_merge = [] + self.packages_to_merge = set() # Словарь для хранения деревьев директорий для различных пакетов. - self.packages_file_trees = {} + self.packages_file_trees = OrderedDict() # Список обработчиков. self._handlers = {} @@ -1764,7 +1764,7 @@ class DirectoryProcessor: if self.for_package: self.output.set_info('Processing packages from merge parameter...') - self.processed_packages.append(self.for_package) + self.processed_packages.add(self.for_package) self._merge_packages() if self._handlers_queue: @@ -1829,9 +1829,18 @@ class DirectoryProcessor: '''Метод для выполнения шаблонов относящихся к пакетам, указанным во всех встреченных значениях параметра merge.''' not_merged_packages = [] + founded_packages = iter(self.packages_file_trees.keys()) while self.packages_to_merge: - self.for_package = self.packages_to_merge.pop() + self.for_package = next(founded_packages, None) + + if self.for_package is None: + # Если список найденных пакетов пройден, но пакеты для + # настройки еще есть -- заново проходимся по списку. + founded_packages = iter(self.packages_file_trees.keys()) + continue + elif self.for_package not in self.packages_to_merge: + continue self.template_engine.for_package = self.for_package if self.for_package not in self.packages_file_trees: @@ -1856,11 +1865,12 @@ class DirectoryProcessor: directory_tree=directory_tree, package=package) - self.processed_packages.append(self.for_package) + self.processed_packages.add(self.for_package) + self.packages_to_merge.remove(self.for_package) if not_merged_packages: self.output.set_error('Packages {} is not merged.'. - format(','.join(self.packages_to_merge))) + format(','.join(not_merged_packages))) else: self.output.set_success('All packages are merged.') @@ -1922,6 +1932,8 @@ class DirectoryProcessor: self.template_engine.change_directory(current_directory_path) template_directories, template_files = self._scan_directory( current_directory_path) + template_files = sorted(template_files) + print('template files:', template_files) # обрабатываем в первую очередь шаблон директории. if '.calculate_directory' in template_files: @@ -1979,7 +1991,9 @@ class DirectoryProcessor: # Если есть параметр merge -- сохраняем присутствующие в нем пакеты # для последующей обработки. if self.for_package and directory_parameters.merge: - self.packages_to_merge.extend(directory_parameters.merge) + for pkg in directory_parameters.merge: + if pkg not in self.processed_packages: + self.packages_to_merge.add(pkg) if directory_parameters.notify: for handler_id in directory_parameters.notify: @@ -2026,6 +2040,9 @@ class DirectoryProcessor: # Просто псевдоним, чтобы меньше путаницы было далее. template_parameters = directory_parameters + print("\nLET'S USE TEMPLATE FILES: {}\n".format(template_files)) + print("THIS PARAMETERS WOULD BE INHERITED BY TEMPLATE FILES:") + print(template_parameters) # Обрабатываем файлы шаблонов. for template_name in template_files: @@ -2056,6 +2073,17 @@ class DirectoryProcessor: # добавляем ее в словарь обработчиков и пропускаем ее. self._handlers.update({template_parameters.handler: (FILE, template_path)}) + print('SKIPPED:', template_path) + + # * * * ПРИДУМАТЬ ОПТИМИЗАЦИЮ * * * + # TODO Потому что накладывать дерево каждый раз, когда + # обнаружены файлы какого-то пакета не рационально. + # Обновляем дерево директорий для данного пакета, если + # происходит его заполнение. + if self.fill_trees: + self._update_package_tree(template_parameters.package, + directory_tree[directory_name]) + directory_tree[directory_name] = {} continue if self._handling is not None and template_parameters.handler: @@ -2068,7 +2096,9 @@ class DirectoryProcessor: # Если есть параметр merge добавляем его содержимое в список # пакетов для последующей обработки. if self.for_package and template_parameters.merge: - self.packages_to_merge.extend(template_parameters.merge) + for pkg in template_parameters.merge: + if pkg not in self.processed_packages: + self.packages_to_merge.add(pkg) if template_parameters.notify: for handler_id in template_parameters.notify: @@ -2098,16 +2128,6 @@ class DirectoryProcessor: if not target_file_path: continue - # * * * ПРИДУМАТЬ ОПТИМИЗАЦИЮ * * * - # TODO Потому что накладывать дерево каждый раз, когда обнаружены - # файлы какого-то пакета не рационально. - # Обновляем дерево директорий для данного пакета, если происходит - # его заполнение. - if self.fill_trees: - self._update_package_tree(template_parameters.package, - directory_tree[directory_name]) - directory_tree[directory_name] = {} - # Проходимся далее по директориям. for directory in template_directories: if self.fill_trees: @@ -2309,9 +2329,6 @@ class DirectoryProcessor: дерево каталогов, то в случае несовпадения значений package для файла или директории, им в дереве присваивается значение None.''' if parameters.handler: - print('\nHANDLER WAS FOUND:', template_path) - print('NOW HANDLERS:', *list(self._handlers.keys())) - print() return False if parameters.append != 'skip' or parameters.action: @@ -2340,9 +2357,12 @@ class DirectoryProcessor: elif parameters.package != self.for_package: if directory_tree is not None: + print('IS ADDED TO TREE') template_name = os.path.basename(template_path) directory_tree[template_name] = None - + print('CURRENT TREE:', self.directory_tree) + else: + print('IS NOT ADDED TO TREE') self.output.set_warning( ("'package' parameter value '{0}' does not " "match its current target package '{1}'. " diff --git a/tests/templates/test_directory_processor.py b/tests/templates/test_directory_processor.py index c69092d..48614ba 100644 --- a/tests/templates/test_directory_processor.py +++ b/tests/templates/test_directory_processor.py @@ -8,6 +8,8 @@ from calculate.utils.package import PackageAtomName, Version, Package from calculate.utils.files import join_paths from calculate.utils.io_module import IOModule from calculate.templates.template_engine import Variables +from collections import OrderedDict +from pprint import pprint CHROOT_PATH = os.path.join(os.getcwd(), @@ -593,7 +595,7 @@ class TestDirectoryProcessor: '/etc/dir_14/file_0')) assert '/etc/dir_14/file_0' not in test_package - def test_if_template_s_directory_contains_two_directories_with_single_template_files_that_belongs_to_a_different_packages_and_target_files_does_not_exist_and_one_of_a_template_file_has_the_merge_parameter_with_other_package_and_directory_processor_is_used_for_a_package__the_directory_processor_creates_one_file_using_template_with_actual_package_parameter_and_then_uses_the_packages_file_tree_to_merge_an_other_package(self): + def test_if_template_s_directory_contains_two_directories_with_single_template_files_that_belongs_to_a_different_packages_and_target_files_does_not_exist_and_one_of_a_template_files_has_the_merge_parameter_with_other_package_and_directory_processor_is_used_for_a_package__the_directory_processor_creates_one_file_using_template_with_actual_package_parameter_and_then_uses_the_packages_file_tree_to_merge_an_other_package(self): datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH, 'templates_18') directory_processor = DirectoryProcessor('install', @@ -975,6 +977,34 @@ class TestDirectoryProcessor: except KeyboardInterrupt: assert False + def test_merge_order(self): + try: + datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH, + 'templates_36') + directory_processor = DirectoryProcessor( + 'install', + datavars_module=datavars, + package='test-category/test-package' + ) + directory_processor.process_template_directories() + + expected_output = {} + real_output = {} + + for number in range(0, 3): + expected_output.update({'file_{}'.format(number): + 'from script_{}'.format(number)}) + output_path = os.path.join(CHROOT_PATH, + 'etc/dir_50/file_{}' + .format(number)) + with open(output_path, 'r') as output_file: + output_text = output_file.read() + real_output.update({'file_{}'.format(number): output_text}) + + assert real_output == expected_output + except KeyboardInterrupt: + assert False + def test_view_tree(self): list_path = join_paths(CHROOT_PATH, '/etc') show_tree(list_path) diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_36/install/.calculate_directory b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/.calculate_directory new file mode 100644 index 0000000..321ca7a --- /dev/null +++ b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/.calculate_directory @@ -0,0 +1 @@ +{% calculate append = 'skip', path = '/etc/dir_50', action = 'install' %} diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_0 b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_0 new file mode 100644 index 0000000..7961aef --- /dev/null +++ b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_0 @@ -0,0 +1,14 @@ +{% calculate run = '/usr/bin/python', package = 'test-category/test-package', +merge = 'test-category/other-package' %} +import os +print('IN script_0') +print('current dir:', os.getcwd()) +files_list = os.listdir('./') +if not files_list: + with open('./file_0', 'w') as new_file: + new_file.write('from script_0') +else: + new_file_path = './file_{}'.format(int(files_list[-1].lstrip('file_')) + 1) + with open(new_file_path, 'w') as new_file: + new_file.write('from script_0') + diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_1 b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_1 new file mode 100644 index 0000000..bd748b0 --- /dev/null +++ b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_1 @@ -0,0 +1,13 @@ +{% calculate run = '/usr/bin/python', package = 'test-category/other-package', +merge = 'test-category/new-package' %} +import os +files_list = os.listdir('./') +if not files_list: + with open('./file_0', 'w') as new_file: + new_file.write('from script_1') +else: + new_file_path = './file_{}'.format(int(files_list[-1].lstrip('file_')) + 1) + print('script_1 creates file:', new_file_path) + with open(new_file_path, 'w') as new_file: + new_file.write('from script_1') + diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_2 b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_2 new file mode 100644 index 0000000..f3a5574 --- /dev/null +++ b/tests/templates/testfiles/test_dir_processor_root/templates_36/install/script_2 @@ -0,0 +1,12 @@ +{% calculate run = '/usr/bin/python', package = 'test-category/new-package' %} +import os +files_list = os.listdir('./') +if not files_list: + with open('./file_0', 'w') as new_file: + new_file.write('from script_2') +else: + new_file_path = './file_{}'.format(int(files_list[-1].lstrip('file_')) + 1) + print('script_2 creates file:', new_file_path) + with open(new_file_path, 'w') as new_file: + new_file.write('from script_2') +