Browse Source

Support for multiple package templates is partially implemented.

packages
Иванов Денис 2 years ago
parent
commit
bf9776f88e
  1. 99
      calculate/templates/template_engine.py
  2. 51
      calculate/templates/template_processor.py
  3. 78
      calculate/utils/package.py
  4. 48
      tests/templates/test_directory_processor.py
  5. 4
      tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/.calculate_directory
  6. 8
      tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/file_0
  7. 4
      tests/templates/testfiles/test_wrapper_root/var/db/pkg/test-category/test-package-1.0/CONTENTS
  8. 14
      tests/utils/test_package.py

99
calculate/templates/template_engine.py

@ -14,7 +14,7 @@ import re
import os
from ..utils.package import PackageAtomParser, PackageAtomError, NOTEXIST,\
Version
Version, PackageAtomName
from ..utils.files import join_paths, check_directory_link, check_command,\
FilesError
from calculate.variables.datavars import HashType, NamespaceNode,\
@ -120,6 +120,8 @@ class ParametersProcessor:
self._inspect_formats_package()
self._for_package = None
# Если добавляемый параметр нуждается в проверке -- добавляем сюда
# метод для проверки.
self.checkers_list = OrderedDict({
@ -152,6 +154,14 @@ class ParametersProcessor:
# указываем здесь эти условия.
self.inherit_conditions = {'chmod': self.is_chmod_inheritable}
@property
def for_package(self):
return self._for_package
@for_package.setter
def for_package(self):
return self._for_package
def set_parameters_container(self, parameters_container):
self._parameters_container = parameters_container
@ -243,24 +253,26 @@ class ParametersProcessor:
# Методы для проверки параметров во время разбора шаблона.
def check_package_parameter(self, parameter_value):
try:
if isinstance(parameter_value, str):
result = self.package_atom_parser.parse_package_parameter(
parameter_value)
elif isinstance(parameter_value, list):
result = []
for atom in parameter_value:
result.append(
self.package_atom_parser.parse_package_parameter(
atom)
)
except PackageAtomError as error:
if error.errno == NOTEXIST:
raise ConditionFailed(error.message, self.lineno)
else:
raise IncorrectParameter(error.message)
if isinstance(parameter_value, str):
parameter_value = [parameter_value]
result = []
unexisting = []
for atom in parameter_value:
try:
result.append(
self.package_atom_parser.parse_package_parameter(atom))
except PackageAtomError as error:
if error.errno == NOTEXIST:
unexisting.append(error.atom_name)
else:
raise IncorrectParameter(error.message)
if not result:
raise ConditionFailed("package{0} '{1}'".format(
's' if len(unexisting) > 1 else '',
"', '".join(unexisting)),
self.lineno)
return result
def check_append_parameter(self, parameter_value):
@ -771,7 +783,8 @@ class CalculateExtension(Extension):
super().__init__(environment)
self.environment = environment
self.environment.globals.update({'pkg': self.pkg})
self.environment.globals.update({'pkg': self.pkg,
'for_pkg': self.for_pkg})
self._datavars = datavars_module
self.parameters_processor = parameters_processor
@ -827,22 +840,25 @@ class CalculateExtension(Extension):
variable_name = nodes.List(variable_name, lineno=lineno)
if self.stream.skip_if('assign'):
return self._make_save_node(variable_name,
target_file,
self.ASSIGN,
lineno)
save_node = self._make_save_node(variable_name,
target_file,
self.ASSIGN,
lineno)
elif self.stream.skip_if('add') and self.stream.skip_if('assign'):
return self._make_save_node(variable_name,
target_file,
self.APPEND,
lineno)
save_node = self._make_save_node(variable_name,
target_file,
self.APPEND,
lineno)
elif self.stream.skip_if('sub') and self.stream.skip_if('assign'):
return self._make_save_node(variable_name,
target_file,
self.REMOVE,
lineno)
raise TemplateSyntaxError("'=' is expected in 'save' tag",
lineno=lineno)
save_node = self._make_save_node(variable_name,
target_file,
self.REMOVE,
lineno)
else:
raise TemplateSyntaxError("'=' is expected in 'save' tag",
lineno=lineno)
return nodes.Output([nodes.Const('')], lineno=lineno)
def parse_calculate(self):
'''Метод для разбора тега calculate, содержащего значения параметров и
@ -902,6 +918,7 @@ class CalculateExtension(Extension):
return nodes.Output([nodes.Const('')], lineno=lineno)
def check_parameter(self, parameter_name, parameter_value, context):
print(f"parameter '{parameter_name}' = {parameter_value}")
self.parameters_processor.check_template_parameter(
parameter_name,
parameter_value,
@ -1241,10 +1258,14 @@ class CalculateExtension(Extension):
return Version()
else:
# package = context.parent['__parameters__'].package
package = self.parameters_processor._parameters_container.package
package = self.parameters_processor._parameters_container.\
package
if not package:
return Version()
return package.version
return package[0].version
def for_pkg(self) -> PackageAtomName:
return self.parameters_processor.for_package or None
class TemplateEngine:
@ -1314,6 +1335,14 @@ class TemplateEngine:
Version=Version
)
@property
def for_package(self):
return self.parameters_processor._for_package
@for_package.setter
def for_package(self, package):
self.parameters_processor._for_package = package
def process_template_from_string(self, string, template_type,
parameters=None):
'''Метод для обработки текста шаблона.'''

51
calculate/templates/template_processor.py

@ -1656,7 +1656,8 @@ class DirectoryProcessor:
elif isinstance(package, str):
try:
self.for_package = self.template_engine.\
parameters_processor.check_package_parameter(package)
parameters_processor.check_package_parameter(
package)[0]
except ConditionFailed as error:
# ConfitionFailed потому что для проверки значения пакета,
# используется тот же метод, что проверяет параметр package
@ -1732,6 +1733,8 @@ class DirectoryProcessor:
else:
package = None
self.template_engine.for_package = self.for_package
for directory_path in self.template_paths:
self.base_directory = directory_path.strip()
entries = os.scandir(self.base_directory)
@ -1774,6 +1777,7 @@ class DirectoryProcessor:
not_merged_packages.append(self.for_package)
continue
self.template_engine.for_package = self.for_package
package = Package(self.for_package,
chroot_path=self.cl_chroot_path)
@ -1882,8 +1886,10 @@ class DirectoryProcessor:
# используя нынешнее состояние дерева директорий, обновляем
# дерево пакета текущего шаблона директории.
if self.fill_trees:
self._update_package_tree(directory_parameters.package,
directory_tree[directory_name])
for pkg in directory_parameters.package:
self._update_package_tree(
pkg,
directory_tree[directory_name])
# Перед выходом из директории очищаем текущий уровень
# дерева.
directory_tree = {}
@ -1899,26 +1905,12 @@ class DirectoryProcessor:
# текущий пакет ветки шаблонов на этот.
if directory_parameters.package:
if (package is None or
package.package_name != directory_parameters.package):
package = Package(directory_parameters.package,
package.package_name not in directory_parameters.package):
package = Package(directory_parameters.package[0],
chroot_path=self.cl_chroot_path)
else:
# Если .calculate_directory отсутствует -- создаем директорию,
# используя унаследованные параметры и имя самой директории.
if not self._check_package_and_action(
directory_parameters,
current_directory_path,
directory_tree=(directory_tree if
self.fill_trees else None)):
# Обновляем дерево директорий для данного пакета.
if self.fill_trees:
self._update_package_tree(directory_parameters.package,
directory_tree[directory_name])
# Перед выходом из директории очищаем текущий уровень
# дерева.
directory_tree = {}
return
# Для того чтобы директория была создана, просто добавляем параметр
# append = join.
directory_parameters.set_parameter({'append': 'join'})
@ -1988,8 +1980,8 @@ class DirectoryProcessor:
template_package = package
if template_parameters.package:
if (template_package is None or
package.package_name != directory_parameters.package):
template_package = Package(directory_parameters.package,
package.package_name not in template_parameters.package):
template_package = Package(template_parameters.package[0],
chroot_path=self.cl_chroot_path)
# Выполняем действия, указанные в шаблоне.
@ -2008,8 +2000,9 @@ class DirectoryProcessor:
# Обновляем дерево директорий для данного пакета, если происходит
# его заполнение.
if self.fill_trees:
self._update_package_tree(template_parameters.package,
directory_tree[directory_name])
for pkg in template_parameters.package:
self._update_package_tree(pkg,
directory_tree[directory_name])
directory_tree[directory_name] = {}
# Проходимся далее по директориям.
@ -2236,16 +2229,18 @@ class DirectoryProcessor:
"'package' parameter is not defined. Template: {}".
format(template_path))
elif parameters.package != self.for_package:
elif self.for_package not in parameters.package:
if directory_tree is not None:
template_name = os.path.basename(template_path)
directory_tree[template_name] = None
self.output.set_warning(
("'package' parameter value '{0}' does not "
"match its current target package '{1}'. "
"Template: {2}").
format(parameters.package.atom,
("'package' parameter value{0} '{1}' does not "
"match its current target package '{2}'. "
"Template: {3}").
format('s' if len(parameters.package) > 1 else '',
"', '".join(pkg.atom for pkg in
parameters.package),
self.for_package.atom,
template_path)
)

78
calculate/utils/package.py

@ -22,9 +22,11 @@ DEFAULT, NOTEXIST, NOTCORRECT = range(3)
class PackageAtomError(Exception):
'''Исключение выбрасываемое при ошибках разбора ATOM-названий.'''
def __init__(self, message='Package atom error', errno=DEFAULT):
def __init__(self, message='Package atom error', errno=DEFAULT,
atom_name=None):
self.message = message
self.errno = errno
self.atom = atom_name
class VersionError(Exception):
@ -206,12 +208,25 @@ class Version:
class PackageAtomName:
'''Класс для хранения результата определения пакета. Для определения пакета
использует путь к его pkg директории.'''
def __init__(self, atom_dictionary):
def __init__(self, atom_dictionary, parser=None):
self._package_directory = atom_dictionary['pkg_path']
self._version = atom_dictionary['version']
if self._package_directory is None:
self._name = 'None'
else:
self._name = atom_dictionary.get('name',
os.path.basename(
self._package_directory))
self._parser = parser
@property
def name(self):
if self._package_directory is None:
return 'None'
return self._name
@property
def fullname(self):
if self._package_directory is None:
return 'None'
return os.path.basename(self._package_directory)
@ -226,7 +241,7 @@ class PackageAtomName:
def atom(self):
if self._package_directory is None:
return 'None'
return "{}/{}".format(self.category, self.name)
return "{}/{}".format(self.category, self.fullname)
@property
def version(self):
@ -243,7 +258,8 @@ class PackageAtomName:
return read_file(use_path).strip('\n').split(' ')
except FilesError:
raise PackageAtomError("could not read use flags for 'package'"
" parameter: {}".format(self.package_atom))
" parameter: {}".format(self.package_atom),
atom_name=self.package_atom)
@property
def pkg_path(self):
@ -258,12 +274,27 @@ class PackageAtomName:
return read_file(slot_path).strip('\n')
except FilesError:
raise PackageAtomError("could not read slot value for"
" 'package': {}".format(self.package_atom))
" 'package': {}".format(self.package_atom),
atom_name=self.package_atom)
def __eq__(self, other):
print('EQ')
print('other:', other)
if isinstance(other, PackageAtomName):
return (self._package_directory ==
other._package_directory)
result = (self._package_directory ==
other._package_directory)
print('result:', result)
return result
elif isinstance(other, str) and self._parser:
try:
other_atom = self._parser.parse_package_parameter(other)
print('OTHER ATOM:', other_atom)
result = (self._package_directory ==
other_atom._package_directory)
print('result:', result)
return result
except Exception:
return False
else:
return False
@ -271,6 +302,12 @@ class PackageAtomName:
if isinstance(other, PackageAtomName):
return (self._package_directory !=
other._package_directory)
elif isinstance(other, str) and self._parser:
try:
other_atom = self._parser.parse_package_parameter(other)
return self._package_directory != other_atom._package_directory
except Exception:
return True
else:
return False
@ -332,7 +369,7 @@ class PackageAtomParser:
not parsing_result.groupdict()['name']):
raise PackageAtomError("'package' parameter value '{}' is not"
" correct".format(package_atom),
errno=NOTCORRECT)
errno=NOTCORRECT, atom_name=self.atom_name)
self._atom_dictionary['category'] = parsing_result.groupdict(
)['category']
@ -359,7 +396,7 @@ class PackageAtomParser:
self._check_package_existance()
atom_name_object = PackageAtomName(self._atom_dictionary)
atom_name_object = PackageAtomName(self._atom_dictionary, parser=self)
self._atom_dictionary.clear()
return atom_name_object
@ -395,7 +432,8 @@ class PackageAtomParser:
raise PackageAtomError("Package from 'package' parameter value"
" '{}' does not exist".format(
self.package_atom),
errno=NOTEXIST)
errno=NOTEXIST,
atom_name=self.package_atom)
if len(glob_result) == 1:
# Если нашелся один пакет.
@ -430,7 +468,8 @@ class PackageAtomParser:
"Package from 'package' parameter value"
" '{}' does not exist".format(
self.package_atom),
errno=NOTEXIST)
errno=NOTEXIST,
atom_name=self.package_atom)
if len(packages) == 1:
# Если был найден только один кандидат -- выдаем его.
@ -454,7 +493,8 @@ class PackageAtomParser:
raise PackageAtomError("Package from 'package' parameter value"
" '{}' does not exist".format(
self.package_atom),
errno=NOTEXIST)
errno=NOTEXIST,
atom_name=self.package_atom)
def _check_use_flags_value(self, pkg_path):
'''Метод для проверки полученных из параметра package значений
@ -468,7 +508,8 @@ class PackageAtomParser:
"Package from 'package' parameter value"
" '{}' does not exist".format(
self.package_atom),
errno=NOTEXIST)
errno=NOTEXIST,
atom_name=self.package_atom)
def _get_slot_value(self, pkg_path):
'''Метод для получения значения slot из файла SLOT.'''
@ -477,7 +518,8 @@ class PackageAtomParser:
return read_file(slot_path).strip('\n')
except FilesError:
raise PackageAtomError("could not read slot value for"
" 'package': {}".format(self.package_atom))
" 'package': {}".format(self.package_atom),
atom_name=self.package_atom)
def _get_use_flags_value(self, pkg_path):
'''Метод для получения списка значений use-флагов из файла USE.'''
@ -486,7 +528,8 @@ class PackageAtomParser:
return read_file(use_path).strip('\n').split(' ')
except FilesError:
raise PackageAtomError("could not read use flags for 'package'"
" parameter: {}".format(self.package_atom))
" parameter: {}".format(self.package_atom),
atom_name=self.package_atom)
def _get_category_packages(self, category):
'''Генератор имен категорий, имеющихся в /var/db/pkg'''
@ -518,7 +561,8 @@ class PackageAtomParser:
package_atom = PackageAtomName(
{'pkg_path': package_path,
'version': version})
'version': version},
parser=self)
return package_atom
except (OSError, IOError):
continue
@ -533,7 +577,7 @@ class PackageAtomParser:
@property
def atom_name(self):
'''Метод для получения из ATOM-словаря объекта PackageAtomName.'''
return PackageAtomName(self._atom_dictionary)
return PackageAtomName(self._atom_dictionary, parser=self)
@atom_dictionary.setter
def set_atom_dictionary(self, atom_dictionary):

48
tests/templates/test_directory_processor.py

@ -894,19 +894,41 @@ class TestDirectoryProcessor:
'etc/dir_35/file_0'))
assert '/etc/dir_35/file_0' in test_package
# def test_multiple_package_value(self):
# datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH,
# 'templates_34')
# directory_processor = DirectoryProcessor(['install', 'update'],
# datavars_module=datavars)
# directory_processor.process_template_directories()
# test_package = Package(test_package_name, chroot_path=CHROOT_PATH)
# assert os.path.exists(join_paths(CHROOT_PATH,
# 'etc/dir_36'))
# assert os.path.exists(join_paths(CHROOT_PATH,
# 'etc/dir_36/file_0'))
# assert '/etc/dir_36/file_0' in test_package
def test_multiple_package_value(self):
test_package = Package(test_package_name, chroot_path=CHROOT_PATH)
other_package = Package(other_package_name, chroot_path=CHROOT_PATH)
datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH,
'templates_34')
directory_processor = DirectoryProcessor(
'install',
datavars_module=datavars,
package='test-category/test-package')
directory_processor.process_template_directories()
assert os.path.exists(join_paths(CHROOT_PATH,
'etc/dir_36'))
# with open(join_paths(CHROOT_PATH,
# 'etc/dir_36/other'), 'r') as file_0:
# text = file_0.read()
# print('OTHER TEXT:')
# print(text)
assert os.path.exists(join_paths(CHROOT_PATH,
'etc/dir_36/test'))
assert '/etc/dir_36/test' in test_package
directory_processor = DirectoryProcessor(
'install',
datavars_module=datavars,
package='test-category/other-package')
directory_processor.process_template_directories()
assert os.path.exists(join_paths(CHROOT_PATH,
'etc/dir_36'))
assert os.path.exists(join_paths(CHROOT_PATH,
'etc/dir_36/other'))
assert '/etc/dir_36/other' in other_package
def test_view_tree(self):
list_path = join_paths(CHROOT_PATH, '/etc')

4
tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/.calculate_directory

@ -1,2 +1,2 @@
{% calculate append = 'skip',
package = ['test-category/test-package', 'test-category/other-package'] %}
{% calculate append = 'join', package = ['test-category/test-package',
'test-category/other-package'] %}

8
tests/templates/testfiles/test_dir_processor_root/templates_34/install/dir_36/file_0

@ -1 +1,7 @@
{% calculate merge = 'test-category/test-package' -%}
{% calculate append = 'join', format = 'bind' -%}
{% calculate name = {'test-package': 'test',
'other-package': 'other'}[for_pkg().name] -%}
options {
parameter-1 {{ variables.variable_1 }};
parameter-2 {{ variables.variable_2 }};
};

4
tests/templates/testfiles/test_wrapper_root/var/db/pkg/test-category/test-package-1.0/CONTENTS

@ -6,6 +6,6 @@ obj /etc/dir/deleted.json 11e3a79fe51cce828d973dba8702adaa 1591356795
dir /etc/dir/subdir
obj /etc/dir/subdir/config.json 506649cf099878124deda2c2452c3693 1591605050
obj /etc/dir/subdir/file 84bcceb2c8e6de79849ea5f3304f2411 1591343236
obj /etc/dir/file.conf 0b87fea7f5b65cac5012baa2bf647e72 1601290690
obj /etc/dir/file.conf 0b87fea7f5b65cac5012baa2bf647e72 1601363686
dir /etc/dir/dir_1
obj /etc/dir/dir_1/config.json 506649cf099878124deda2c2452c3693 1601290690
obj /etc/dir/dir_1/config.json 506649cf099878124deda2c2452c3693 1601363686

14
tests/utils/test_package.py

@ -141,7 +141,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
'dev-lang/python-3.6.9')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,
@ -152,7 +152,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
'dev-lang/python-3.6:3.6/3.6m')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,
@ -163,7 +163,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
atom_name = package_atom.parse_package_parameter(
'dev-lang/python-3.6:')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,
@ -174,7 +174,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
'dev-lang/python-3.6 [abi_x86_64 ssl]')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,
@ -187,7 +187,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
'dev-lang/python:3.6/3.6m')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,
@ -199,7 +199,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
'dev-lang/python [specific_flag]')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,
@ -220,7 +220,7 @@ sym /etc/test_dir_2/symlink -> file_2.cfg {}
atom_name = package_atom.parse_package_parameter('dev-lang/python')
assert (atom_name.category == 'dev-lang' and
atom_name.name == 'python-3.6.9' and
atom_name.fullname == 'python-3.6.9' and
atom_name.version == Version('3.6.9') and
atom_name.pkg_path == join_paths(
CHROOT_PATH,

Loading…
Cancel
Save