From bf9776f88e11bb1cc1ae588d3fbdcaef776bd92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=81?= Date: Tue, 29 Sep 2020 17:36:48 +0300 Subject: [PATCH] Support for multiple package templates is partially implemented. --- calculate/templates/template_engine.py | 99 ++++++++++++------- calculate/templates/template_processor.py | 51 +++++----- calculate/utils/package.py | 78 +++++++++++---- tests/templates/test_directory_processor.py | 48 ++++++--- .../install/dir_36/.calculate_directory | 4 +- .../templates_34/install/dir_36/file_0 | 8 +- .../test-category/test-package-1.0/CONTENTS | 4 +- tests/utils/test_package.py | 14 +-- 8 files changed, 201 insertions(+), 105 deletions(-) diff --git a/calculate/templates/template_engine.py b/calculate/templates/template_engine.py index 3532120..a41ce8f 100644 --- a/calculate/templates/template_engine.py +++ b/calculate/templates/template_engine.py @@ -14,7 +14,7 @@ import re import os from ..utils.package import PackageAtomParser, PackageAtomError, NOTEXIST,\ - Version + Version, PackageAtomName from ..utils.files import join_paths, check_directory_link, check_command,\ FilesError from calculate.variables.datavars import HashType, NamespaceNode,\ @@ -120,6 +120,8 @@ class ParametersProcessor: self._inspect_formats_package() + self._for_package = None + # Если добавляемый параметр нуждается в проверке -- добавляем сюда # метод для проверки. self.checkers_list = OrderedDict({ @@ -152,6 +154,14 @@ class ParametersProcessor: # указываем здесь эти условия. self.inherit_conditions = {'chmod': self.is_chmod_inheritable} + @property + def for_package(self): + return self._for_package + + @for_package.setter + def for_package(self): + return self._for_package + def set_parameters_container(self, parameters_container): self._parameters_container = parameters_container @@ -243,24 +253,26 @@ class ParametersProcessor: # Методы для проверки параметров во время разбора шаблона. def check_package_parameter(self, parameter_value): - try: - if isinstance(parameter_value, str): - result = self.package_atom_parser.parse_package_parameter( - parameter_value) - elif isinstance(parameter_value, list): - result = [] - for atom in parameter_value: - result.append( - self.package_atom_parser.parse_package_parameter( - atom) - ) - - except PackageAtomError as error: - if error.errno == NOTEXIST: - raise ConditionFailed(error.message, self.lineno) - else: - raise IncorrectParameter(error.message) + if isinstance(parameter_value, str): + parameter_value = [parameter_value] + + result = [] + unexisting = [] + for atom in parameter_value: + try: + result.append( + self.package_atom_parser.parse_package_parameter(atom)) + except PackageAtomError as error: + if error.errno == NOTEXIST: + unexisting.append(error.atom_name) + else: + raise IncorrectParameter(error.message) + if not result: + raise ConditionFailed("package{0} '{1}'".format( + 's' if len(unexisting) > 1 else '', + "', '".join(unexisting)), + self.lineno) return result def check_append_parameter(self, parameter_value): @@ -771,7 +783,8 @@ class CalculateExtension(Extension): super().__init__(environment) self.environment = environment - self.environment.globals.update({'pkg': self.pkg}) + self.environment.globals.update({'pkg': self.pkg, + 'for_pkg': self.for_pkg}) self._datavars = datavars_module self.parameters_processor = parameters_processor @@ -827,22 +840,25 @@ class CalculateExtension(Extension): variable_name = nodes.List(variable_name, lineno=lineno) if self.stream.skip_if('assign'): - return self._make_save_node(variable_name, - target_file, - self.ASSIGN, - lineno) + save_node = self._make_save_node(variable_name, + target_file, + self.ASSIGN, + lineno) elif self.stream.skip_if('add') and self.stream.skip_if('assign'): - return self._make_save_node(variable_name, - target_file, - self.APPEND, - lineno) + save_node = self._make_save_node(variable_name, + target_file, + self.APPEND, + lineno) elif self.stream.skip_if('sub') and self.stream.skip_if('assign'): - return self._make_save_node(variable_name, - target_file, - self.REMOVE, - lineno) - raise TemplateSyntaxError("'=' is expected in 'save' tag", - lineno=lineno) + save_node = self._make_save_node(variable_name, + target_file, + self.REMOVE, + lineno) + else: + raise TemplateSyntaxError("'=' is expected in 'save' tag", + lineno=lineno) + + return nodes.Output([nodes.Const('')], lineno=lineno) def parse_calculate(self): '''Метод для разбора тега calculate, содержащего значения параметров и @@ -902,6 +918,7 @@ class CalculateExtension(Extension): return nodes.Output([nodes.Const('')], lineno=lineno) def check_parameter(self, parameter_name, parameter_value, context): + print(f"parameter '{parameter_name}' = {parameter_value}") self.parameters_processor.check_template_parameter( parameter_name, parameter_value, @@ -1241,10 +1258,14 @@ class CalculateExtension(Extension): return Version() else: # package = context.parent['__parameters__'].package - package = self.parameters_processor._parameters_container.package + package = self.parameters_processor._parameters_container.\ + package if not package: return Version() - return package.version + return package[0].version + + def for_pkg(self) -> PackageAtomName: + return self.parameters_processor.for_package or None class TemplateEngine: @@ -1314,6 +1335,14 @@ class TemplateEngine: Version=Version ) + @property + def for_package(self): + return self.parameters_processor._for_package + + @for_package.setter + def for_package(self, package): + self.parameters_processor._for_package = package + def process_template_from_string(self, string, template_type, parameters=None): '''Метод для обработки текста шаблона.''' diff --git a/calculate/templates/template_processor.py b/calculate/templates/template_processor.py index 3e313f8..25f3358 100644 --- a/calculate/templates/template_processor.py +++ b/calculate/templates/template_processor.py @@ -1656,7 +1656,8 @@ class DirectoryProcessor: elif isinstance(package, str): try: self.for_package = self.template_engine.\ - parameters_processor.check_package_parameter(package) + parameters_processor.check_package_parameter( + package)[0] except ConditionFailed as error: # ConfitionFailed потому что для проверки значения пакета, # используется тот же метод, что проверяет параметр package @@ -1732,6 +1733,8 @@ class DirectoryProcessor: else: package = None + self.template_engine.for_package = self.for_package + for directory_path in self.template_paths: self.base_directory = directory_path.strip() entries = os.scandir(self.base_directory) @@ -1774,6 +1777,7 @@ class DirectoryProcessor: not_merged_packages.append(self.for_package) continue + self.template_engine.for_package = self.for_package package = Package(self.for_package, chroot_path=self.cl_chroot_path) @@ -1882,8 +1886,10 @@ class DirectoryProcessor: # используя нынешнее состояние дерева директорий, обновляем # дерево пакета текущего шаблона директории. if self.fill_trees: - self._update_package_tree(directory_parameters.package, - directory_tree[directory_name]) + for pkg in directory_parameters.package: + self._update_package_tree( + pkg, + directory_tree[directory_name]) # Перед выходом из директории очищаем текущий уровень # дерева. directory_tree = {} @@ -1899,26 +1905,12 @@ class DirectoryProcessor: # текущий пакет ветки шаблонов на этот. if directory_parameters.package: if (package is None or - package.package_name != directory_parameters.package): - package = Package(directory_parameters.package, + package.package_name not in directory_parameters.package): + package = Package(directory_parameters.package[0], chroot_path=self.cl_chroot_path) else: # Если .calculate_directory отсутствует -- создаем директорию, # используя унаследованные параметры и имя самой директории. - if not self._check_package_and_action( - directory_parameters, - current_directory_path, - directory_tree=(directory_tree if - self.fill_trees else None)): - # Обновляем дерево директорий для данного пакета. - if self.fill_trees: - self._update_package_tree(directory_parameters.package, - directory_tree[directory_name]) - # Перед выходом из директории очищаем текущий уровень - # дерева. - directory_tree = {} - return - # Для того чтобы директория была создана, просто добавляем параметр # append = join. directory_parameters.set_parameter({'append': 'join'}) @@ -1988,8 +1980,8 @@ class DirectoryProcessor: template_package = package if template_parameters.package: if (template_package is None or - package.package_name != directory_parameters.package): - template_package = Package(directory_parameters.package, + package.package_name not in template_parameters.package): + template_package = Package(template_parameters.package[0], chroot_path=self.cl_chroot_path) # Выполняем действия, указанные в шаблоне. @@ -2008,8 +2000,9 @@ class DirectoryProcessor: # Обновляем дерево директорий для данного пакета, если происходит # его заполнение. if self.fill_trees: - self._update_package_tree(template_parameters.package, - directory_tree[directory_name]) + for pkg in template_parameters.package: + self._update_package_tree(pkg, + directory_tree[directory_name]) directory_tree[directory_name] = {} # Проходимся далее по директориям. @@ -2236,16 +2229,18 @@ class DirectoryProcessor: "'package' parameter is not defined. Template: {}". format(template_path)) - elif parameters.package != self.for_package: + elif self.for_package not in parameters.package: if directory_tree is not None: template_name = os.path.basename(template_path) directory_tree[template_name] = None self.output.set_warning( - ("'package' parameter value '{0}' does not " - "match its current target package '{1}'. " - "Template: {2}"). - format(parameters.package.atom, + ("'package' parameter value{0} '{1}' does not " + "match its current target package '{2}'. " + "Template: {3}"). + format('s' if len(parameters.package) > 1 else '', + "', '".join(pkg.atom for pkg in + parameters.package), self.for_package.atom, template_path) ) diff --git a/calculate/utils/package.py b/calculate/utils/package.py index 5eddcf0..31f57a1 100644 --- a/calculate/utils/package.py +++ b/calculate/utils/package.py @@ -22,9 +22,11 @@ DEFAULT, NOTEXIST, NOTCORRECT = range(3) class PackageAtomError(Exception): '''Исключение выбрасываемое при ошибках разбора ATOM-названий.''' - def __init__(self, message='Package atom error', errno=DEFAULT): + def __init__(self, message='Package atom error', errno=DEFAULT, + atom_name=None): self.message = message self.errno = errno + self.atom = atom_name class VersionError(Exception): @@ -206,12 +208,25 @@ class Version: class PackageAtomName: '''Класс для хранения результата определения пакета. Для определения пакета использует путь к его pkg директории.''' - def __init__(self, atom_dictionary): + def __init__(self, atom_dictionary, parser=None): self._package_directory = atom_dictionary['pkg_path'] self._version = atom_dictionary['version'] + if self._package_directory is None: + self._name = 'None' + else: + self._name = atom_dictionary.get('name', + os.path.basename( + self._package_directory)) + self._parser = parser @property def name(self): + if self._package_directory is None: + return 'None' + return self._name + + @property + def fullname(self): if self._package_directory is None: return 'None' return os.path.basename(self._package_directory) @@ -226,7 +241,7 @@ class PackageAtomName: def atom(self): if self._package_directory is None: return 'None' - return "{}/{}".format(self.category, self.name) + return "{}/{}".format(self.category, self.fullname) @property def version(self): @@ -243,7 +258,8 @@ class PackageAtomName: return read_file(use_path).strip('\n').split(' ') except FilesError: raise PackageAtomError("could not read use flags for 'package'" - " parameter: {}".format(self.package_atom)) + " parameter: {}".format(self.package_atom), + atom_name=self.package_atom) @property def pkg_path(self): @@ -258,12 +274,27 @@ class PackageAtomName: return read_file(slot_path).strip('\n') except FilesError: raise PackageAtomError("could not read slot value for" - " 'package': {}".format(self.package_atom)) + " 'package': {}".format(self.package_atom), + atom_name=self.package_atom) def __eq__(self, other): + print('EQ') + print('other:', other) if isinstance(other, PackageAtomName): - return (self._package_directory == - other._package_directory) + result = (self._package_directory == + other._package_directory) + print('result:', result) + return result + elif isinstance(other, str) and self._parser: + try: + other_atom = self._parser.parse_package_parameter(other) + print('OTHER ATOM:', other_atom) + result = (self._package_directory == + other_atom._package_directory) + print('result:', result) + return result + except Exception: + return False else: return False @@ -271,6 +302,12 @@ class PackageAtomName: if isinstance(other, PackageAtomName): return (self._package_directory != other._package_directory) + elif isinstance(other, str) and self._parser: + try: + other_atom = self._parser.parse_package_parameter(other) + return self._package_directory != other_atom._package_directory + except Exception: + return True else: return False @@ -332,7 +369,7 @@ class PackageAtomParser: not parsing_result.groupdict()['name']): raise PackageAtomError("'package' parameter value '{}' is not" " correct".format(package_atom), - errno=NOTCORRECT) + errno=NOTCORRECT, atom_name=self.atom_name) self._atom_dictionary['category'] = parsing_result.groupdict( )['category'] @@ -359,7 +396,7 @@ class PackageAtomParser: self._check_package_existance() - atom_name_object = PackageAtomName(self._atom_dictionary) + atom_name_object = PackageAtomName(self._atom_dictionary, parser=self) self._atom_dictionary.clear() return atom_name_object @@ -395,7 +432,8 @@ class PackageAtomParser: raise PackageAtomError("Package from 'package' parameter value" " '{}' does not exist".format( self.package_atom), - errno=NOTEXIST) + errno=NOTEXIST, + atom_name=self.package_atom) if len(glob_result) == 1: # Если нашелся один пакет. @@ -430,7 +468,8 @@ class PackageAtomParser: "Package from 'package' parameter value" " '{}' does not exist".format( self.package_atom), - errno=NOTEXIST) + errno=NOTEXIST, + atom_name=self.package_atom) if len(packages) == 1: # Если был найден только один кандидат -- выдаем его. @@ -454,7 +493,8 @@ class PackageAtomParser: raise PackageAtomError("Package from 'package' parameter value" " '{}' does not exist".format( self.package_atom), - errno=NOTEXIST) + errno=NOTEXIST, + atom_name=self.package_atom) def _check_use_flags_value(self, pkg_path): '''Метод для проверки полученных из параметра package значений @@ -468,7 +508,8 @@ class PackageAtomParser: "Package from 'package' parameter value" " '{}' does not exist".format( self.package_atom), - errno=NOTEXIST) + errno=NOTEXIST, + atom_name=self.package_atom) def _get_slot_value(self, pkg_path): '''Метод для получения значения slot из файла SLOT.''' @@ -477,7 +518,8 @@ class PackageAtomParser: return read_file(slot_path).strip('\n') except FilesError: raise PackageAtomError("could not read slot value for" - " 'package': {}".format(self.package_atom)) + " 'package': {}".format(self.package_atom), + atom_name=self.package_atom) def _get_use_flags_value(self, pkg_path): '''Метод для получения списка значений use-флагов из файла USE.''' @@ -486,7 +528,8 @@ class PackageAtomParser: return read_file(use_path).strip('\n').split(' ') except FilesError: raise PackageAtomError("could not read use flags for 'package'" - " parameter: {}".format(self.package_atom)) + " parameter: {}".format(self.package_atom), + atom_name=self.package_atom) def _get_category_packages(self, category): '''Генератор имен категорий, имеющихся в /var/db/pkg''' @@ -518,7 +561,8 @@ class PackageAtomParser: package_atom = PackageAtomName( {'pkg_path': package_path, - 'version': version}) + 'version': version}, + parser=self) return package_atom except (OSError, IOError): continue @@ -533,7 +577,7 @@ class PackageAtomParser: @property def atom_name(self): '''Метод для получения из ATOM-словаря объекта PackageAtomName.''' - return PackageAtomName(self._atom_dictionary) + return PackageAtomName(self._atom_dictionary, parser=self) @atom_dictionary.setter def set_atom_dictionary(self, atom_dictionary): diff --git a/tests/templates/test_directory_processor.py b/tests/templates/test_directory_processor.py index c246f44..27b6a2d 100644 --- a/tests/templates/test_directory_processor.py +++ b/tests/templates/test_directory_processor.py @@ -894,19 +894,41 @@ class TestDirectoryProcessor: 'etc/dir_35/file_0')) assert '/etc/dir_35/file_0' in test_package - # def test_multiple_package_value(self): - # datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH, - # 'templates_34') - # directory_processor = DirectoryProcessor(['install', 'update'], - # datavars_module=datavars) - # directory_processor.process_template_directories() - # test_package = Package(test_package_name, chroot_path=CHROOT_PATH) - - # assert os.path.exists(join_paths(CHROOT_PATH, - # 'etc/dir_36')) - # assert os.path.exists(join_paths(CHROOT_PATH, - # 'etc/dir_36/file_0')) - # assert '/etc/dir_36/file_0' in test_package + def test_multiple_package_value(self): + test_package = Package(test_package_name, chroot_path=CHROOT_PATH) + other_package = Package(other_package_name, chroot_path=CHROOT_PATH) + + datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH, + 'templates_34') + directory_processor = DirectoryProcessor( + 'install', + datavars_module=datavars, + package='test-category/test-package') + directory_processor.process_template_directories() + + assert os.path.exists(join_paths(CHROOT_PATH, + 'etc/dir_36')) + # with open(join_paths(CHROOT_PATH, + # 'etc/dir_36/other'), 'r') as file_0: + # text = file_0.read() + # print('OTHER TEXT:') + # print(text) + + assert os.path.exists(join_paths(CHROOT_PATH, + 'etc/dir_36/test')) + assert '/etc/dir_36/test' in test_package + + directory_processor = DirectoryProcessor( + 'install', + datavars_module=datavars, + package='test-category/other-package') + directory_processor.process_template_directories() + + assert os.path.exists(join_paths(CHROOT_PATH, + 'etc/dir_36')) + assert os.path.exists(join_paths(CHROOT_PATH, + 'etc/dir_36/other')) + assert '/etc/dir_36/other' in other_package def test_view_tree(self): list_path = join_paths(CHROOT_PATH, '/etc') diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/.calculate_directory b/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/.calculate_directory index 9c40f92..4989453 100644 --- a/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/.calculate_directory +++ b/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/.calculate_directory @@ -1,2 +1,2 @@ -{% calculate append = 'skip', -package = ['test-category/test-package', 'test-category/other-package'] %} +{% calculate append = 'join', package = ['test-category/test-package', + 'test-category/other-package'] %} diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/file_0 b/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/file_0 index d253f96..244e696 100644 --- a/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/file_0 +++ b/tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/file_0 @@ -1 +1,7 @@ -{% calculate merge = 'test-category/test-package' -%} +{% calculate append = 'join', format = 'bind' -%} +{% calculate name = {'test-package': 'test', + 'other-package': 'other'}[for_pkg().name] -%} +options { + parameter-1 {{ variables.variable_1 }}; + parameter-2 {{ variables.variable_2 }}; +}; diff --git a/tests/templates/testfiles/test_wrapper_root/var/db/pkg/test-category/test-package-1.0/CONTENTS b/tests/templates/testfiles/test_wrapper_root/var/db/pkg/test-category/test-package-1.0/CONTENTS index 263b533..241d43c 100644 --- a/tests/templates/testfiles/test_wrapper_root/var/db/pkg/test-category/test-package-1.0/CONTENTS +++ b/tests/templates/testfiles/test_wrapper_root/var/db/pkg/test-category/test-package-1.0/CONTENTS @@ -6,6 +6,6 @@ obj /etc/dir/deleted.json 11e3a79fe51cce828d973dba8702adaa 1591356795 dir /etc/dir/subdir obj /etc/dir/subdir/config.json 506649cf099878124deda2c2452c3693 1591605050 obj /etc/dir/subdir/file 84bcceb2c8e6de79849ea5f3304f2411 1591343236 -obj /etc/dir/file.conf 0b87fea7f5b65cac5012baa2bf647e72 1601290690 +obj /etc/dir/file.conf 0b87fea7f5b65cac5012baa2bf647e72 1601363686 dir /etc/dir/dir_1 -obj /etc/dir/dir_1/config.json 506649cf099878124deda2c2452c3693 1601290690 +obj /etc/dir/dir_1/config.json 506649cf099878124deda2c2452c3693 1601363686 diff --git a/tests/utils/test_package.py b/tests/utils/test_package.py index ec102bc..ab17991 100644 --- a/tests/utils/test_package.py +++ b/tests/utils/test_package.py @@ -141,7 +141,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} 'dev-lang/python-3.6.9') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH, @@ -152,7 +152,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} 'dev-lang/python-3.6:3.6/3.6m') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH, @@ -163,7 +163,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} atom_name = package_atom.parse_package_parameter( 'dev-lang/python-3.6:') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH, @@ -174,7 +174,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} 'dev-lang/python-3.6 [abi_x86_64 ssl]') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH, @@ -187,7 +187,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} 'dev-lang/python:3.6/3.6m') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH, @@ -199,7 +199,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} 'dev-lang/python [specific_flag]') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH, @@ -220,7 +220,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {} atom_name = package_atom.parse_package_parameter('dev-lang/python') assert (atom_name.category == 'dev-lang' and - atom_name.name == 'python-3.6.9' and + atom_name.fullname == 'python-3.6.9' and atom_name.version == Version('3.6.9') and atom_name.pkg_path == join_paths( CHROOT_PATH,