Browse Source

contents format is executable now.

master
Иванов Денис 7 months ago
parent
commit
998a495669
100 changed files with 894 additions and 499 deletions
  1. +1
    -0
      .gitignore
  2. +0
    -0
      TODO.txt
  3. +3
    -9
      calculate/templates/format/compiz_format.py
  4. +189
    -143
      calculate/templates/format/contents_format.py
  5. +4
    -3
      calculate/templates/format/patch_format.py
  6. +26
    -4
      calculate/templates/template_engine.py
  7. +17
    -7
      calculate/templates/template_processor.py
  8. +14
    -0
      calculate/utils/contents_template
  9. +184
    -49
      calculate/utils/package.py
  10. +2
    -0
      calculate/vars/main/__init__.py
  11. +5
    -0
      calculate/vars/main/cl/groups/__init__.py
  12. +82
    -0
      regex_exp.py
  13. +6
    -4
      tests/templates/format/test_base.py
  14. +5
    -4
      tests/templates/format/test_bind.py
  15. +3
    -3
      tests/templates/format/test_compiz.py
  16. +212
    -157
      tests/templates/format/test_contents.py
  17. +3
    -3
      tests/templates/format/test_dovecot.py
  18. +6
    -3
      tests/templates/format/test_json.py
  19. +4
    -4
      tests/templates/format/test_kde.py
  20. +3
    -3
      tests/templates/format/test_kernel.py
  21. +4
    -4
      tests/templates/format/test_ldap.py
  22. +3
    -3
      tests/templates/format/test_openrc.py
  23. +15
    -17
      tests/templates/format/test_patch.py
  24. +7
    -5
      tests/templates/format/test_postfix.py
  25. +3
    -3
      tests/templates/format/test_procmail.py
  26. +6
    -3
      tests/templates/format/test_proftpd.py
  27. +5
    -4
      tests/templates/format/test_samba.py
  28. +29
    -10
      tests/templates/format/test_xml_gconf.py
  29. +10
    -5
      tests/templates/format/test_xml_xfce.py
  30. +0
    -0
      tests/templates/format/testfiles/base/__init__.py
  31. +0
    -0
      tests/templates/format/testfiles/base/logic_lines_test.txt
  32. +0
    -0
      tests/templates/format/testfiles/base/logic_lines_test_input.txt
  33. +0
    -0
      tests/templates/format/testfiles/base/logic_lines_test_output.txt
  34. +0
    -0
      tests/templates/format/testfiles/bind/__init__.py
  35. +0
    -0
      tests/templates/format/testfiles/bind/original.conf
  36. +0
    -0
      tests/templates/format/testfiles/bind/result.conf
  37. +0
    -0
      tests/templates/format/testfiles/bind/template.conf
  38. +0
    -0
      tests/templates/format/testfiles/compiz/__init__.py
  39. +0
    -0
      tests/templates/format/testfiles/compiz/original
  40. +0
    -0
      tests/templates/format/testfiles/compiz/result
  41. +0
    -0
      tests/templates/format/testfiles/compiz/template
  42. +0
    -0
      tests/templates/format/testfiles/contents/__init__.py
  43. +1
    -0
      tests/templates/format/testfiles/contents/root_0.backup/etc/dir_0/file_0
  44. +1
    -0
      tests/templates/format/testfiles/contents/root_0.backup/etc/dir_0/file_1
  45. +1
    -0
      tests/templates/format/testfiles/contents/root_0.backup/etc/dir_1/file_0
  46. +1
    -0
      tests/templates/format/testfiles/contents/root_0.backup/etc/dir_2/file_0
  47. +1
    -0
      tests/templates/format/testfiles/contents/root_0.backup/etc/dir_3/link_0
  48. +3
    -0
      tests/templates/format/testfiles/contents/root_0.backup/var/db/pkg/test-category/test-package-1.0/CONTENTS
  49. +12
    -0
      tests/templates/format/testfiles/contents/root_1.backup/var/db/pkg/test-category/test-package-1.0/CONTENTS
  50. +11
    -0
      tests/templates/format/testfiles/contents/root_2.backup/var/db/pkg/test-category/other-package-1.1/CONTENTS
  51. +1
    -0
      tests/templates/format/testfiles/contents/root_2.backup/var/db/pkg/test-category/other-package-1.1/SLOT
  52. +11
    -0
      tests/templates/format/testfiles/contents/root_2.backup/var/db/pkg/test-category/test-package-1.0/CONTENTS
  53. +0
    -18
      tests/templates/format/testfiles/contents_original
  54. +0
    -22
      tests/templates/format/testfiles/contents_result
  55. +0
    -9
      tests/templates/format/testfiles/contents_template
  56. +0
    -0
      tests/templates/format/testfiles/dovecot/__init__.py
  57. +0
    -0
      tests/templates/format/testfiles/dovecot/original.conf
  58. +0
    -0
      tests/templates/format/testfiles/dovecot/result.conf
  59. +0
    -0
      tests/templates/format/testfiles/dovecot/template.conf
  60. +0
    -0
      tests/templates/format/testfiles/json/__init__.py
  61. +0
    -0
      tests/templates/format/testfiles/json/original.json
  62. +0
    -0
      tests/templates/format/testfiles/json/result.json
  63. +0
    -0
      tests/templates/format/testfiles/json/template.json
  64. +0
    -0
      tests/templates/format/testfiles/kde/__init__.py
  65. +0
    -0
      tests/templates/format/testfiles/kde/original
  66. +0
    -0
      tests/templates/format/testfiles/kde/result
  67. +0
    -0
      tests/templates/format/testfiles/kde/template
  68. +0
    -0
      tests/templates/format/testfiles/kernel/__init__.py
  69. +0
    -0
      tests/templates/format/testfiles/kernel/original
  70. +0
    -0
      tests/templates/format/testfiles/kernel/result
  71. +0
    -0
      tests/templates/format/testfiles/kernel/template
  72. +0
    -0
      tests/templates/format/testfiles/ldap/__init__.py
  73. +0
    -0
      tests/templates/format/testfiles/ldap/logic_lines_test.txt
  74. +0
    -0
      tests/templates/format/testfiles/ldap/original.conf
  75. +0
    -0
      tests/templates/format/testfiles/ldap/result.conf
  76. +0
    -0
      tests/templates/format/testfiles/ldap/template.conf
  77. +0
    -0
      tests/templates/format/testfiles/openrc/__init__.py
  78. +0
    -0
      tests/templates/format/testfiles/openrc/original
  79. +0
    -0
      tests/templates/format/testfiles/openrc/result
  80. +0
    -0
      tests/templates/format/testfiles/openrc/template
  81. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/a/dir/file1.txt
  82. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/a/dir/file2.txt
  83. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/a1/dir/file1.txt
  84. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/b/dir/file1.txt
  85. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/b/dir/file2.txt
  86. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/b1/dir/file1.txt
  87. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/b1/dir/file2.txt
  88. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/diff_1.patch
  89. +0
    -0
      tests/templates/format/testfiles/patch/root.backup/diff_2.patch
  90. +0
    -0
      tests/templates/format/testfiles/postfix/__init__.py
  91. +0
    -0
      tests/templates/format/testfiles/postfix/original
  92. +0
    -0
      tests/templates/format/testfiles/postfix/result
  93. +0
    -0
      tests/templates/format/testfiles/postfix/template
  94. +0
    -0
      tests/templates/format/testfiles/procmail/__init__.py
  95. +0
    -0
      tests/templates/format/testfiles/procmail/original
  96. +0
    -0
      tests/templates/format/testfiles/procmail/result
  97. +0
    -0
      tests/templates/format/testfiles/procmail/template
  98. +0
    -0
      tests/templates/format/testfiles/proftpd/__init__.py
  99. +0
    -0
      tests/templates/format/testfiles/proftpd/original.conf
  100. +0
    -0
      tests/templates/format/testfiles/proftpd/result.conf

+ 1
- 0
.gitignore View File

@@ -12,3 +12,4 @@ venv/
ENV/
env.bak/
venv.bak/
__pycache__/

calculate/templates/__pycache__/template_processor.cpython-36.pyc.140556154662960 → TODO.txt View File


+ 3
- 9
calculate/templates/format/compiz_format.py View File

@@ -5,7 +5,7 @@ from ..template_engine import ParametersContainer
from collections import OrderedDict
from pyparsing import originalTextFor, Literal, ZeroOrMore, Word, printables,\
OneOrMore, alphanums, ParseException, restOfLine,\
pyparsing_unicode, Group, Optional
pyparsing_unicode, Group, Optional, Regex


class CompizFormat(Format):
@@ -73,10 +73,7 @@ class CompizFormat(Format):
+ section_name('name')
+ Literal(']').suppress())('section_name')

parameter_name = originalTextFor(
OneOrMore(Word(printables,
excludeChars='='))
)
parameter_name = originalTextFor(OneOrMore(Regex('[^=]+')))

parameter_value = Word(printables)

@@ -93,10 +90,7 @@ class CompizFormat(Format):

cls._comment_line = originalTextFor(
Literal(cls.comment_symbol)
+ ZeroOrMore(Word(
printables
+ pyparsing_unicode.alphanums))
)('comment')
+ Regex(r'.*'))('comment')
cls._initialized = True

def _parse_section_line(self, line):


+ 189
- 143
calculate/templates/format/contents_format.py View File

@@ -1,176 +1,222 @@
# vim: fileencoding=utf-8
#
import os
from .base_format import Format
from collections import OrderedDict
from jinja2 import PackageLoader, Environment
from pyparsing import Literal, Regex, Word, nums, alphanums, Optional,\
ParseException
from pyparsing import Literal, Regex, SkipTo, LineEnd, lineno, LineStart
from calculate.utils.package import PackageAtomParser, Package, PackageNotFound
from calculate.utils.files import join_paths
from pprint import pprint
from glob import iglob
from fnmatch import fnmatch


ADD, REMOVE, MOVE = range(0, 3)


class ContentsFormat(Format):
FORMAT = 'contents'
EXECUTABLE = False
EXECUTABLE = True

_initialized = False

comment_symbol = '#'

def __new__(cls, *args, **kwargs):
'''Метод для инициализации парсеров.'''
if not cls._initialized:
cls._initialize_parser()
return super().__new__(cls)
return super().__new__(cls)

def __init__(self, document_text: str,
template_path,
ignore_comments=False,
join_before=False,
add_header=False,
already_changed=False,
parameters=None,
template_parser=True):
processing_methods = [self._parse_dir_line,
self._parse_sym_line,
self._parse_obj_line]

super().__init__(processing_methods)

self._ignore_comments = ignore_comments
self._join_before = join_before

# флаг для указания режима, в котором работает формат.
# Если True => формат работает для наложения шаблонов;
# Если False => формат работает для получения словаря, используемого
# модулем package.
self._template_parser_flag = template_parser

if not self._initialized:
self._initialize_parser()

if (add_header and not ignore_comments and template_parser
and template_path is not None):
self.header, document_text = self._get_header_and_document_text(
document_text,
template_path,
already_changed=already_changed)
else:
self.header = ''
ignore_comments=False):
self._command_methods = {ADD: self._add_command,
REMOVE: self._remove_command,
MOVE: self._move_command}
self._errors = []

self._commands = self._parse_template(document_text)
self._template_path = template_path
self._packages = dict()
self._atom_parser = None

def execute_format(self, target_path, chroot_path='/'):
'''Метод для запуска работы формата.'''
self._package = dict()
self._atom_parser = PackageAtomParser(chroot_path=chroot_path)
for command in self._commands:
target_package = self._get_package(command['target'], chroot_path)

if 'source' not in command:
source_package = None
else:
source_package = self._get_package(command['source'],
chroot_path)

self._command_methods[command['action']](target_package,
command['path'],
command['lineno'],
source=source_package)
pprint(target_package.contents_dictionary)
self._save_changes()

def _add_command(self, target, glob_path, lineno, source=None):
# Если файл уже есть в пакете -- ничего не делаем.
glob_path = join_paths(target.chroot_path, glob_path)
for file_path in iglob(glob_path):
# Если файл уже в пакете -- ничего не делаем.
if file_path in target:
continue

# Проверяем существование файла.
if not os.path.exists(file_path):
error = (f"Format processing error:{lineno}: file "
f"'{file_path}' does not exist.")
self._errors.append(error)
continue

# Проверяем, не принадлежит ли файл какому-нибудь другому пакету.
try:
file_package = self._atom_parser.get_file_package(file_path)
except PackageNotFound:
file_package = None
if file_package is not None:
error = (f"Format processing error:{lineno}: can not add file"
f" '{file_path}' belonging to the package"
f" {file_package} into the package '{target}'.")
self._errors.append(error)
continue

target.add_file(file_path)

def _remove_command(self, target, glob_path, lineno, source=None):
paths_to_remove = []
if glob_path.endswith('/'):
glob_path = glob_path[0:-1]

for file_path in target.contents_dictionary.keys():
if fnmatch(file_path, glob_path):
paths_to_remove.append(file_path)

for file_path in paths_to_remove:
print(file_path)
target.remove_file(file_path)

def _move_command(self, target, glob_path, lineno, source=None):
if source is None:
return

document_text = document_text.strip()
if document_text == '':
self._document_dictionary = OrderedDict()
paths_to_move = []
if glob_path.endswith('/'):
glob_path = glob_path[0:-1]

for file_path in source.contents_dictionary.keys():
if fnmatch(file_path, glob_path):
paths_to_move.append(file_path)

for file_path in paths_to_move:
removed = source.remove_file(file_path)

for file_path, file_info in removed.items():
if file_info['type'] == 'dir':
target.add_dir(file_path)
elif file_info['type'] == 'obj':
target.add_obj(file_path,
file_md5=file_info['md5'],
mtime=file_info['mtime'])
elif file_info['type'] == 'sym':
target.add_sym(file_path,
target_path=file_info['target'],
mtime=file_info['mtime'])
target.sort_contents_dictionary()

def _save_changes(self):
for atom_name, package in self._packages.items():
print(f"SAVING CHANGES FOR PACKAGE: {package}")
print('CONTENTS TEXT:')
print(package.render_contents_file())
package.write_contents_file()
self._packages = {}

def _get_package(self, atom_name: str, chroot_path: str) -> Package:
atom = self._atom_parser.parse_package_parameter(atom_name)
if atom not in self._packages:
package = Package(atom, chroot_path=chroot_path)
self._packages.update({atom: package})
else:
document_lines = self._get_list_of_logic_lines(document_text)
self._lines_to_dictionary(document_lines)
package = self._packages[atom]

return package

def _parse_template(self, template_text: str) -> list:
for tokens, start, end in self.template_parser.scanString(
template_text):
parse_result = tokens[0]
if parse_result is None:
continue
elif 'error' in parse_result:
error = (f'Parse Error:{parse_result["lineno"]}:'
f' {parse_result["error"]}')
self._errors.append(error)
continue
yield parse_result

@classmethod
def _initialize_parser(cls):
action_symbols = (Literal('!') | Literal('-'))

sym_keyword = Literal('sym')
dir_keyword = Literal('dir')
obj_keyword = Literal('obj')

symlink_arrow = Literal('->')
atom = Regex(PackageAtomParser.atom_regex)
remove_symbol = Literal('!')
path = Regex(r'\S+')

file_path = Regex(r'\S+')
add_parser = atom('target') + path('path') + LineEnd().suppress()
add_parser.setParseAction(cls._parse_add)

time_value = Word(nums)
move_parser = (atom('source') + Literal(',').suppress()
+ atom('target') + path('path') + LineEnd().suppress())
move_parser.setParseAction(cls._parse_move)

md5 = Word(alphanums)
remove_parser = (remove_symbol.suppress() + atom('target')
+ path('path') + LineEnd().suppress())
remove_parser.setParseAction(cls._parse_remove)

cls.sym_line = (Optional(action_symbols, default='')('action')
+ sym_keyword('type') + file_path('name')
+ symlink_arrow.suppress() + file_path('target')
+ time_value('time'))
empty_line = LineStart().suppress() + LineEnd().suppress()
empty_line.setParseAction(cls._parse_skip)

cls.dir_line = (Optional(action_symbols, default='')('action')
+ dir_keyword('type') + file_path('name'))
comment = Literal('#') + SkipTo(LineEnd(), include=True)
comment.setParseAction(cls._parse_skip)

cls.obj_line = (Optional(action_symbols, default='')('action')
+ obj_keyword('type') + file_path('name')
+ md5('md5') + time_value('time'))
unexpected = SkipTo(LineEnd(), include=True)
unexpected.setParseAction(cls._parse_unexpected)

cls._initialized = True
cls.template_parser = (move_parser | remove_parser | add_parser
| empty_line | comment | unexpected)

def _parse_sym_line(self, line):
'''Метод для разбора строк типа sym.'''
try:
parsing_result = self.sym_line.parseString(line)
self._match = True
if self._template_parser_flag:
output_name = (parsing_result.action, parsing_result.name)
output_value = (parsing_result.type,
parsing_result.target,
parsing_result.time)
self._item_to_add = OrderedDict({output_name: [output_value]})
else:
output_name = parsing_result.name
output_value = OrderedDict({'type': parsing_result.type,
'target': parsing_result.target,
'mtime': parsing_result.time})

self._item_to_add = OrderedDict({output_name: output_value})
self._ready_to_update = True
except ParseException:
return
@classmethod
def _parse_add(cls, string, location, parse_result):
target = parse_result.asDict()['target']
path = parse_result.asDict()['path']
return {'action': ADD, 'target': target, 'path': path,
'lineno': lineno(location, string)}

def _parse_dir_line(self, line):
'''Метод для разбора строк типа dir.'''
try:
parsing_result = self.dir_line.parseString(line)
self._match = True
if self._template_parser_flag:
output_name = (parsing_result.action, parsing_result.name)
output_value = (parsing_result.type,)
self._item_to_add = OrderedDict({output_name: [output_value]})
else:
output_name = parsing_result.name
output_value = OrderedDict({'type': parsing_result.type})
@classmethod
def _parse_remove(cls, string, location, parse_result):
target = parse_result.asDict()['target']
path = parse_result.asDict()['path']
return {'action': REMOVE, 'target': target, 'path': path,
'lineno': lineno(location, string)}

self._item_to_add = OrderedDict({output_name: output_value})
self._ready_to_update = True
except ParseException:
return
@classmethod
def _parse_move(cls, string, location, parse_result):
source = parse_result.asDict()['source']
target = parse_result.asDict()['target']
path = parse_result.asDict()['path']
return {'action': MOVE, 'source': source,
'target': target, 'path': path,
'lineno': lineno(location, string)}

def _parse_obj_line(self, line):
'''Метод для разбора строк типа obj.'''
try:
parsing_result = self.obj_line.parseString(line)
self._match = True
if self._template_parser_flag:
output_name = (parsing_result.action, parsing_result.name)
output_value = (parsing_result.type,
parsing_result.md5,
parsing_result.time)
self._item_to_add = OrderedDict({output_name: [output_value]})
else:
output_name = parsing_result.name
output_value = OrderedDict({'type': parsing_result.type,
'md5': parsing_result.md5,
'mtime': parsing_result.time})
self._item_to_add = OrderedDict({output_name: output_value})

self._ready_to_update = True
except ParseException:
return
@classmethod
def _parse_skip(cls, parse_result):
return [None]

@property
def document_text(self):
'''Метод для получения текста результирующего документа.
Представляет собой переопределение соответствующего метода базового
класса, посколько в данном случае нужно учитывать режим работы формата.
'''
file_loader = PackageLoader('calculate.templates.format',
self.TEMPLATES_DIRECTORY)
formats_environment = Environment(loader=file_loader,
trim_blocks=True,
lstrip_blocks=True)
formats_environment.globals.update(zip=zip)
formats_environment.add_extension('jinja2.ext.do')
template = formats_environment.get_template(self.FORMAT)
document_text = template.render(
document_dictionary=self._document_dictionary,
template_parser=self._template_parser_flag
)
return '{}{}'.format(self.header, document_text)
@classmethod
def _parse_unexpected(cls, string, location, parse_result):
result = parse_result[0]
if not result:
return [None]
return {'error': result, 'lineno': lineno(location, string)}

+ 4
- 3
calculate/templates/format/patch_format.py View File

@@ -11,10 +11,11 @@ class PatchFormat(Format):
FORMAT = 'patch'
EXECUTABLE = True

def __init__(self, document_text: str,
join_before=False,
def __init__(self, patch_text: str,
template_path: str,
ignore_comments=None,
parameters=ParametersContainer()):
self._patch_text = document_text
self._patch_text = patch_text
self._cwd_path = '/'
self._last_level = 0



+ 26
- 4
calculate/templates/template_engine.py View File

@@ -115,8 +115,10 @@ class ParametersProcessor:
chmod_value_regular = re.compile(
r'([r-][w-][x-])([r-][w-][x-])([r-][w-][x-])')

def __init__(self, parameters_container=None, chroot_path='/',
datavars_module=Variables()):
def __init__(self, parameters_container=None,
chroot_path='/',
datavars_module=Variables(),
for_package=None):
self.chroot_path = chroot_path

self.template_type = DIR
@@ -129,6 +131,8 @@ class ParametersProcessor:

self._inspect_formats_package()

self._for_package = None

# Если добавляемый параметр нуждается в проверке -- добавляем сюда
# метод для проверки.
self.checkers_list = OrderedDict({
@@ -168,6 +172,14 @@ class ParametersProcessor:
def set_parameters_container(self, parameters_container):
self._parameters_container = parameters_container

@property
def for_package(self):
return self._for_package

@for_package.setter
def for_package(self, package):
self._for_package = package

def __getattr__(self, parameter_name):
if parameter_name not in self.available_parameters:
raise IncorrectParameter("Unknown parameter: '{}'".
@@ -1321,7 +1333,8 @@ class TemplateEngine:
def __init__(self, directory_path=None,
datavars_module=Variables(),
appends_set=set(),
chroot_path='/'):
chroot_path='/',
for_package=None):
ParametersProcessor._inspect_formats_package()

CalculateExtension._parameters_set =\
@@ -1335,7 +1348,8 @@ class TemplateEngine:
self._parameters_object = ParametersContainer()
self.parameters_processor = ParametersProcessor(
chroot_path=chroot_path,
datavars_module=datavars_module)
datavars_module=datavars_module,
for_package=for_package)

if directory_path is not None:
self.environment = Environment(
@@ -1352,6 +1366,14 @@ class TemplateEngine:

self.environment.context_class = CalculateContext

@property
def for_package(self):
return self.parameters_processor.for_package

@for_package.setter
def for_package(self, package):
self.parameters_processor.for_package = package

def change_directory(self, directory_path):
'''Метод для смены директории в загрузчике.'''
self.environment.loader = FileSystemLoader(directory_path)


+ 17
- 7
calculate/templates/template_processor.py View File

@@ -1672,6 +1672,7 @@ class DirectoryProcessor:
# в шаблонах, а в них этот параметр играет роль условия.
self.output.set_error(str(error))
return
self.template_engine.for_package = self.for_package

# Получаем список директорий шаблонов.
# TODO переменная список.
@@ -1775,8 +1776,9 @@ class DirectoryProcessor:
self.template_executor.save_changes()

def _execute_handlers(self):
'''Метод для запуска обработчиков добавленных в очередь обработчиков
с помощью параметра notify.'''
self.output.set_info('Processing handlers...')
print('CURRENT HANDLERS QUEUE:', ', '.join(self._handlers_queue))
index = 0
while index < len(self._handlers_queue):
handler = self._handlers_queue[index]
@@ -1796,9 +1798,6 @@ class DirectoryProcessor:
self.cl_chroot_path,
ParametersContainer(),
directory_tree=self.directory_tree)
self.output.set_success(
f"Processed handler directory '{handler}'."
f"Path: '{handler_path}'")
elif handler_type is FILE:
handler_dir, handler_name = os.path.split(handler_path)
self.template_engine.change_directory(handler_dir)
@@ -1825,9 +1824,6 @@ class DirectoryProcessor:
self._execute_template(target_file_path, parameters,
FILE, handler_path,
template_text=handler_text)
self.output.set_success(
f"Processed handler file '{handler}'."
f" Path: '{handler_path}'")

def _merge_packages(self):
'''Метод для выполнения шаблонов относящихся к пакетам, указанным во
@@ -1836,6 +1832,7 @@ class DirectoryProcessor:

while self.packages_to_merge:
self.for_package = self.packages_to_merge.pop()
self.template_engine.for_package = self.for_package

if self.for_package not in self.packages_file_trees:
self.output.set_error(
@@ -2360,8 +2357,21 @@ class DirectoryProcessor:

@contextmanager
def _start_handling(self, handler):
'''Метод для перевода обработчика каталогов в режим обработки
хэндлеров.'''
try:
self._handling = handler
yield self
finally:
self._handling = None

@contextmanager
def _set_current_package(self, package):
'''Метод для указания в шаблонизаторе пакета, для которого в данный
момент проводим настройку. Пока не используется.'''
try:
last_package = self.template_engine.for_package
self.template_engine.for_package = package
yield self
finally:
self.template_engine.for_package = last_package

+ 14
- 0
calculate/utils/contents_template View File

@@ -0,0 +1,14 @@
{% for file_name, contents_values in document_dictionary.items() %}
{% if contents_values is mapping %}
{% set contents_values = (contents_values.values()|list) %}
{% else %}
{% set contents_values = (contents_values[0]|list) %}
{% set file_name = file_name[-1] %}
{% endif %}
{% if contents_values[0] == 'sym' %}
{{ contents_values[0] }} {{ file_name }} -> {{ contents_values[1:]|join(' ') }}
{% else %}
{{ contents_values[0] }} {{ file_name }}{% if contents_values[1:] %} {{ contents_values[1:]|join(' ') }}{% endif %}

{% endif %}
{% endfor %}

+ 184
- 49
calculate/utils/package.py View File

@@ -4,8 +4,11 @@ import os
import re
import glob
from collections import OrderedDict
from ..templates.format.contents_format import ContentsFormat
from .files import read_file, read_link, join_paths, FilesError
from pyparsing import Literal, Regex, Word, nums, alphanums,\
LineEnd, SkipTo
from jinja2 import PackageLoader, Environment
from calculate.utils.tools import Singleton
import hashlib
import operator

@@ -203,6 +206,63 @@ class Version:
return True


class ContentsParser(metaclass=Singleton):
def __init__(self):
'''Метод для инициализации парсеров.'''
sym_keyword = Literal('sym')
dir_keyword = Literal('dir')
obj_keyword = Literal('obj')

symlink_arrow = Literal('->')

file_path = Regex(r'\S+')

time_value = Word(nums)

md5 = Word(alphanums)

sym_line = (sym_keyword('type') + file_path('path')
+ symlink_arrow.suppress() + file_path('target')
+ time_value('mtime') + LineEnd().suppress())

dir_line = (dir_keyword('type') + file_path('path')
+ LineEnd().suppress())

obj_line = (obj_keyword('type') + file_path('path')
+ md5('md5') + time_value('mtime') + LineEnd().suppress())
unexpected = SkipTo(LineEnd(), include=True)

self._parser = (dir_line | sym_line |
obj_line | unexpected('unexpected')
).setParseAction(self._parser_method)

def _parser_method(self, parse_result):
if parse_result.getName() == 'unexpected':
return [None]
result_dict = parse_result.asDict()
path = result_dict.pop('path')
value = ({path: result_dict})
return value

def parse(self, contents_text: str):
output_dictionary = OrderedDict()
for tokens, start, end in self._parser.scanString(contents_text):
parse_result = tokens[0]
if parse_result is None:
continue
output_dictionary.update(parse_result)
return output_dictionary

def render(self, contents_dictionary: dict):
file_loader = PackageLoader('calculate', 'utils')
environment = Environment(loader=file_loader,
trim_blocks=True,
lstrip_blocks=True)
template = environment.get_template('contents_template')
text = template.render(document_dictionary=contents_dictionary)
return text


class PackageAtomName:
'''Класс для хранения результата определения пакета. Для определения пакета
использует путь к его pkg директории.'''
@@ -234,6 +294,12 @@ class PackageAtomName:
return 'None'
return self._version

@property
def contents_path(self):
if self._package_directory is None:
return 'None'
return os.path.join(self._package_directory, 'CONTENTS')

@property
def use_flags(self):
if self._package_directory is None:
@@ -296,15 +362,15 @@ class PackageAtomParser:
'''Класс для парсинга параметра package, его проверки, а также определения
принадлежности файла пакету.'''
package_name_pattern =\
r'(?P<name>\D[\w\d]*(\-\D[\w\d]*)*)(?P<version>-\d[^\s:]*)?'
atom_regex = re.compile(r'''(?P<category>[^\s/]*)/
{0}
(?P<slot>:[^\s\[]*)?\s*
(?P<use_flags>\[\S*(?:\s+\S*)*\])?'''.format(
package_name_pattern
),
re.VERBOSE)
r'(?P<name>\D[\w\d]*(\-\D[\w\d]*)*)(?P<version>-\d[^\s,\[:]*)?'
atom_name_pattern = r'''(?P<category>[^\s/]*)/
{0}
(?P<slot>:[^\[\s]*)?
(?P<use_flags>\[\S*(?:,\S*)*\])?'''.format(
package_name_pattern)
atom_regex = re.compile(atom_name_pattern, re.VERBOSE)
package_name_regex = re.compile(package_name_pattern)

def __init__(self, pkg_path='/var/db/pkg',
@@ -354,7 +420,7 @@ class PackageAtomParser:
use_flags = parsing_result.groupdict()['use_flags'].strip().\
rstrip(']').lstrip('[')

for use_flag in use_flags.split():
for use_flag in use_flags.split(','):
self._atom_dictionary['use_flags'].append(use_flag.strip())

self._check_package_existance()
@@ -461,6 +527,8 @@ class PackageAtomParser:
use-флагов.'''
if 'use_flags' in self._atom_dictionary:
use_flags = self._get_use_flags_value(pkg_path)
print('current use flags:', use_flags)
print('atom use flags:', self._atom_dictionary['use_flags'])

for use_flag in self._atom_dictionary['use_flags']:
if use_flag not in use_flags:
@@ -588,6 +656,7 @@ class Package:

self.contents_file_path = self._get_contents_path(package_atom)
self.package_name = package_atom
self.parser = ContentsParser()
if (chroot_path != '/' and
not self.contents_file_path.startswith(chroot_path)):
self.contents_file_path = join_paths(chroot_path,
@@ -638,29 +707,62 @@ class Package:
raise PackageError(str(error))

if contents_text:
contents_format = ContentsFormat(contents_text,
None,
template_parser=False)
self.contents_dictionary = contents_format._document_dictionary
self.contents_dictionary = self.parser.parse(contents_text)
return True
else:
return False

def write_contents_file(self):
'''Метод для записи файла CONTENTS.'''
contents_file = open(self.contents_file_path, 'w')
contents_text = self.render_contents_file()
contents_file.write(contents_text)
contents_file.close()
with open(self.contents_file_path, 'w') as contents_file:
contents_text = self.render_contents_file()
contents_file.write(contents_text)

def render_contents_file(self):
'''Метод для получения текста файла CONTENTS.'''
contents_format = ContentsFormat('', None, template_parser=False)
contents_format._document_dictionary = self.contents_dictionary
return contents_format.document_text
return self.parser.render(self.contents_dictionary)

@property
def files(self):
'''Метод для получения списка путей файлов, имеющихся в CONTENTS-файле
пакета.'''
return list(self.contents_dictionary.keys())

def get_file_type(self, file_path: str) -> str:
'''Метод для получения по пути файла типа, указанного для него в
CONTENTS-файле.'''
file_path = self.remove_chroot_path(file_path)
if file_path in self.contents_dictionary:
return self.contents_dictionary[file_path]['type']
return None

def sort_contents_dictionary(self):
tree = {}
for path in self.contents_dictionary.keys():
path = path.strip('/').split('/')
level = tree
for part in path:
if part not in level:
level[part] = {}
level = level[part]

sorted_contents = OrderedDict()
for path in self._make_paths('/', tree):
sorted_contents[path] = self.contents_dictionary[path]
self.contents_dictionary = sorted_contents

def _make_paths(self, path, level):
paths = []
for part in sorted(level.keys()):
part_path = os.path.join(path, part)
paths.append(part_path)
if level[part]:
paths.extend(self._make_paths(part_path, level[part]))
return paths

def add_dir(self, file_name):
'''Метод для добавления в CONTENTS директорий.'''
print('ADD DIR')
file_name = self.remove_chroot_path(file_name)

if (file_name != '/' and
@@ -670,8 +772,9 @@ class Package:
contents_item = OrderedDict({'type': 'dir'})
self.contents_dictionary[file_name] = contents_item

def add_sym(self, file_name, target_path=None):
def add_sym(self, file_name, target_path=None, mtime=None):
'''Метод для добавления в CONTENTS символьных ссылок.'''
print('ADD SYM')
file_name = self.remove_cfg_prefix(file_name)

real_path = file_name
@@ -681,23 +784,23 @@ class Package:
real_path = join_paths(self.chroot_path, file_name)

if target_path is None:
target = read_link(real_path)
else:
target = target_path
target_path = read_link(real_path)

self.add_dir(os.path.dirname(file_name))
mtime = str(int(os.lstat(real_path).st_mtime))
if mtime is None:
mtime = str(int(os.lstat(real_path).st_mtime))
try:
contents_item = OrderedDict({'type': 'sym',
'target': target,
'target': target_path,
'mtime': mtime})
except FilesError as error:
raise PackageError(str(error))

self.contents_dictionary[file_name] = contents_item

def add_obj(self, file_name, file_md5=None):
def add_obj(self, file_name, file_md5=None, mtime=None):
'''Метод для добавления в CONTENTS обычных файлов как obj.'''
print('ADD OBJ')
real_path = file_name
file_name = self.remove_chroot_path(file_name)
file_name = self.remove_cfg_prefix(file_name)
@@ -705,33 +808,36 @@ class Package:
if real_path == file_name:
real_path = join_paths(self.chroot_path, file_name)
self.add_dir(os.path.dirname(file_name))
if file_md5 is not None:
md5_value = file_md5
else:

if file_md5 is None:
try:
file_text = read_file(real_path).encode()
except FilesError as error:
raise PackageError(str(error))
md5_value = hashlib.md5(file_text).hexdigest()
file_md5 = hashlib.md5(file_text).hexdigest()

if mtime is None:
mtime = str(int(os.lstat(real_path).st_mtime))

contents_item = OrderedDict({'type': 'obj',
'md5': md5_value,
'mtime':
str(int(os.lstat(real_path).st_mtime))})
'md5': file_md5,
'mtime': mtime})
self.contents_dictionary[file_name] = contents_item

def add_file(self, file_name):
'''Метод для добавления в CONTENTS файла любого типа.'''
print('ADD FILE:')
if file_name != '/':
real_path = file_name
file_name = self.remove_chroot_path(file_name)
if real_path != file_name:
if file_name.startswith(self.chroot_path):
file_name = self.remove_chroot_path(file_name)
else:
real_path = join_paths(self.chroot_path, file_name)

if os.path.isdir(real_path):
self.add_dir(file_name)
elif os.path.islink(real_path):
self.add_link(file_name)
self.add_sym(file_name)
elif os.path.isfile(real_path):
self.add_obj(file_name)

@@ -739,28 +845,50 @@ class Package:
'''Метод для удаления файлов и ссылок.'''
file_path = self.remove_chroot_path(file_path)
file_path = self.remove_cfg_prefix(file_path)
removed = OrderedDict()

if file_path in self.contents_dictionary:
self.contents_dictionary.pop(file_path)
removed.update({file_path:
self.contents_dictionary.pop(file_path)})
return removed

def remove_dir(self, file_path):
'''Метод для удаления из CONTENTS файлов и директорий находящихся
внутри удаляемой директории и самой директории.'''
directory_path = self.remove_chroot_path(file_path)
paths_to_remove = []
removed = OrderedDict()

for file_path in self.contents_dictionary:
if file_path.startswith(directory_path):
paths_to_remove.append(file_path)

for file_path in paths_to_remove:
self.contents_dictionary.pop(file_path)
removed.update({file_path:
self.contents_dictionary.pop(file_path)})

return removed

def remove_file(self, file_path):
file_path = self.remove_chroot_path(file_path)
file_path = self.remove_cfg_prefix(file_path)
removed = OrderedDict()

if file_path not in self.contents_dictionary:
return
if self.contents_dictionary[file_path]['type'] == 'dir':
removed.update(self.remove_dir(file_path))
else:
removed.update({file_path:
self.contents_dictionary.pop(file_path)})
return removed

def clear_dir(self, file_path):
'''Метод для удаления из CONTENTS файлов и директорий находящихся
внутри очищаемой директории.'''
directory_path = self.remove_chroot_path(file_path)
paths_to_remove = []
removed = OrderedDict()

for file_path in self.contents_dictionary:
if file_path == directory_path:
@@ -769,13 +897,16 @@ class Package:
paths_to_remove.append(file_path)

for file_path in paths_to_remove:
self.contents_dictionary.pop(file_path)
removed.update({file_path:
self.contents_dictionary.pop(file_path)})
return removed

def remove_empty_directories(self):
'''Метод для удаления из CONTENTS директорий, которые после удаления
тех или иных файлов больше не находятся на пути к тем файлам, которые
по-прежнему принадлежат пакету.'''
used_directories = set()
removed = OrderedDict()
not_directory_list = [path for path, value in
self.contents_dictionary.items()
if value['type'] != 'dir']
@@ -791,7 +922,9 @@ class Package:
file_path not in used_directories]

for file_path in paths_to_delete:
self.contents_dictionary.pop(file_path)
removed.update({file_path:
self.contents_dictionary.pop(file_path)})
return removed

def get_md5(self, file_path):
'''Метод для получения md5 хэш-суммы указанного файла.'''
@@ -817,11 +950,13 @@ class Package:
return file_md5 == contents_md5

def __contains__(self, file_path):
if self.chroot_path != "/" and file_path.startswith(self.chroot_path):
file_path = file_path[len(self.chroot_path):]
file_path = self.remove_cfg_prefix(file_path)

return file_path in self.contents_dictionary
if self.chroot_path != "/":
if file_path.startswith(self.chroot_path):
file_path = file_path[len(self.chroot_path):]
file_path = self.remove_cfg_prefix(file_path)
return file_path in self.contents_dictionary
else:
return True

def __repr__(self):
return '<Package: {}/{}>'.format(self.package_name.category,


+ 2
- 0
calculate/vars/main/__init__.py View File

@@ -7,6 +7,8 @@ main:

Variable('cl_chroot_path', type=StringType.readonly, source='/')

Variable('cl_root_path', type=StringType.readonly, source='/')

Variable('cl_template_path', type=ListType.readonly,
source=['./tests/templates/testfiles/test_runner'])



+ 5
- 0
calculate/vars/main/cl/groups/__init__.py View File

@@ -0,0 +1,5 @@
from calculate.variables.datavars import Variable, TableType

Variable('build', type=TableType, source=[])

Variable('unistall', type=TableType, source=[])

+ 82
- 0
regex_exp.py View File

@@ -0,0 +1,82 @@
from calculate.utils.package import PackageAtomParser
from pyparsing import Regex, Literal, LineEnd, restOfLine, LineStart


ADD, REMOVE, MOVE = range(0, 3)


def parse_add(parse_result):
target = parse_result.asDict()['target']
path = parse_result.asDict()['path']
return {'action': ADD, 'target': target, 'path': path}


def parse_remove(parse_result):
target = parse_result.asDict()['target']
path = parse_result.asDict()['path']
return {'action': REMOVE, 'target': target, 'path': path}


def parse_move(parse_result):
source = parse_result.asDict()['source']
target = parse_result.asDict()['target']
path = parse_result.asDict()['path']
return {'action': MOVE, 'source': source, 'target': target, 'path': path}


def parse_skip(parse_result):
print('skip:', parse_result)
return [None]


def parse_unexpected(parse_result):
print('unexpected')
result = parse_result[0]
if not result:
return [None]
else:
return {'unexpected': result}


atom = Regex(PackageAtomParser.atom_regex)
remove_symbol = Literal('!')
path = Regex(r'\S+')

add_parser = atom('target') + path('path') + LineEnd().suppress()
add_parser.setParseAction(parse_add)

move_parser = (atom('source') + Literal(',').suppress() + atom('target')
+ path('path') + LineEnd().suppress())
move_parser.setParseAction(parse_move)

remove_parser = (remove_symbol.suppress() + atom('target') + path('path')
+ LineEnd().suppress())
remove_parser.setParseAction(parse_remove)

empty_line = LineStart().suppress() + LineEnd().suppress()
empty_line.setParseAction(parse_skip)

comment = Literal('#') + restOfLine + LineEnd().suppress()
comment.setParseAction(parse_skip)

unexpected = restOfLine + LineEnd().suppress()
unexpected.setParseAction(parse_unexpected)


template_parser = (move_parser | remove_parser | add_parser | comment
| empty_line | unexpected)

text = '''dev-lang/python-3.6[abi_x86_64,ssl] /usr/dir/file

# comment
!sys-fs/udisks-1.0.5-r1 /usr/bin/file

ll
dev-lang/python-3.6[abi_x86_64,ssl], sys-fs/udisks-1.0.5-r1 /usr/bin/file

'''

for tokens, start, end in template_parser.scanString(text):
result = tokens[0]
if result is not None:
print(result)

+ 6
- 4
tests/templates/format/test_base.py View File

@@ -56,7 +56,8 @@ class TestJoinMethod:
@pytest.mark.base
class TestLogicLinesMethod:
def test_if_input_is_text_document_the_method_returns_list_of_its_lines(self):
with open('./tests/templates/format/testfiles/logic_lines_test.txt', 'r') as InputFile:
with open('./tests/templates/format/testfiles/base/'
'logic_lines_test.txt', 'r') as InputFile:
InputText = InputFile.read()

processingMethods = []
@@ -69,12 +70,13 @@ class TestLogicLinesMethod:
InputLines = BaseObject._get_list_of_logic_lines(InputText)
assert InputLines == OutputLines


def test_if_lines_in_document_divided_using_backslash_as_continuation_symbol__method_returns_list_of_full_lines(self):
with open('./tests/templates/format/testfiles/logic_lines_test_input.txt', 'r') as InputFile:
with open('./tests/templates/format/testfiles/base/'
'logic_lines_test_input.txt', 'r') as InputFile:
InputText = InputFile.read()

with open('./tests/templates/format/testfiles/logic_lines_test_output.txt', 'r') as OutputFile:
with open('./tests/templates/format/testfiles/base/'
'logic_lines_test_output.txt', 'r') as OutputFile:
OutputText = OutputFile.read()

BaseObject = Format([])


+ 5
- 4
tests/templates/format/test_bind.py View File

@@ -332,15 +332,15 @@ options {
assert bind_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/bind_original.conf',
with open('./tests/templates/format/testfiles/bind/original.conf',
'r') as original_file:
original_text = original_file.read()
print(original_text)
bind_original_object = BINDFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/bind_template.conf',
'r') as template_file:
with open('./tests/templates/format/testfiles/bind/'
'template.conf', 'r') as template_file:
template_text = template_file.read()
bind_template_object = BINDFormat(template_text,
'/path/to/template',
@@ -348,7 +348,8 @@ options {

bind_original_object.join_template(bind_template_object)

with open('./tests/templates/format/testfiles/bind_result.conf', 'r') as result_file:
with open('./tests/templates/format/testfiles/bind/'
'result.conf', 'r') as result_file:
result_text = result_file.read()

assert bind_original_object.document_text == result_text


+ 3
- 3
tests/templates/format/test_compiz.py View File

@@ -202,13 +202,13 @@ class TestParsingMethods:
assert compiz_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/compiz_original',
with open('./tests/templates/format/testfiles/compiz/original',
'r') as original_file:
original_text = original_file.read()
compiz_original_object = CompizFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/compiz_template',
with open('./tests/templates/format/testfiles/compiz/template',
'r') as template_file:
template_text = template_file.read()
compiz_template_object = CompizFormat(template_text,
@@ -217,7 +217,7 @@ class TestParsingMethods:

compiz_original_object.join_template(compiz_template_object)

with open('./tests/templates/format/testfiles/compiz_result',
with open('./tests/templates/format/testfiles/compiz/result',
'r') as result_file:
result_text = result_file.read()



+ 212
- 157
tests/templates/format/test_contents.py View File

@@ -1,180 +1,235 @@
import os
import pytest
import shutil
from collections import OrderedDict
from calculate.templates.format.contents_format import ContentsFormat
from calculate.utils.package import PackageAtomName, Version, Package


TESTFILES_PATH = os.path.join(os.getcwd(),
'tests/templates/format/testfiles/contents')


@pytest.mark.contents
class TestParsingMethods:
def test_if_input_document_contains_a_few_lines_without_any_action_symbols__the_initialised_object_contains_correct_dictionary(self):
document_text = '''
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
'''
class TestExecutingFormat:
def test_copy_testfiles(self):
entries = os.scandir(TESTFILES_PATH)
for root_entry in entries:
if root_entry.path.endswith('.backup'):
shutil.copytree(root_entry.path,
root_entry.path.rstrip('.backup'),
symlinks=True)

def test_if_template_of_the_contents_format_contains_some_files_to_add_directory_to_add_and_link_to_add_and_some_files_is_listed_using_glob_symbols__all_files_links_and_directories_are_added_to_the_target_package(self):
chroot_path = os.path.join(TESTFILES_PATH, 'root_0')
test_package_name = PackageAtomName(
{'pkg_path':
os.path.join(
chroot_path,
'var/db/pkg/test-category/test-package-1.0'),
'version': Version('1.0')})

result = OrderedDict({('', '/usr/bin/vim'):
[('obj', '30acc0f256e11c1ecdb1bd80b688d238',
'1573538056')],
('', '/usr/share/bash-completion/completions/ex'):
[('sym', 'vim', '1573538054')],
('', '/usr/share/bash-completion/completions'):
[('dir',)]})

contents_object = ContentsFormat(document_text, '/path/to/template')
assert result == contents_object._document_dictionary

def test_if_input_document_contains_a_few_lines_with_some_action_symbols__the_initialised_object_contains_correct_dictionary(self):
document_text = '''
!obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
-sym /usr/share/bash-completion/completions/ex -> vim 1573538054
!dir /usr/share/bash-completion/completions
'''
template_text = '''
test-category/test-package /etc/dir_0/*

result = OrderedDict({('!', '/usr/bin/vim'):
[('obj', '30acc0f256e11c1ecdb1bd80b688d238',
'1573538056')],
('-', '/usr/share/bash-completion/completions/ex'):
[('sym', 'vim', '1573538054')],
('!', '/usr/share/bash-completion/completions'):
[('dir',)]})

contents_object = ContentsFormat(document_text, '/path/to/template')
assert result == contents_object._document_dictionary

def test_if_template_parser_flag_is_set_False__the_initialized_object_contains_correct_dictionary_for_contents_util_module(self):
document_text = '''
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
'''
test-category/test-package /etc/dir_1/file_0

result = OrderedDict({'/usr/bin/vim':
OrderedDict(
{'type': 'obj',
'md5': '30acc0f256e11c1ecdb1bd80b688d238',
'mtime': '1573538056'}),
'/usr/share/bash-completion/completions/ex':
OrderedDict(
{'type': 'sym',
'target': 'vim',
'mtime': '1573538054'}),
'/usr/share/bash-completion/completions':
OrderedDict({'type': 'dir'})})

contents_object = ContentsFormat(document_text, '/path/to/template',
template_parser=False)
assert result == contents_object._document_dictionary

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/contents_original',
'r') as original_file:
original_text = original_file.read()
contents_original_object = ContentsFormat(original_text, '/path/to/template')

with open('./tests/templates/format/testfiles/contents_template',
'r') as template_file:
template_text = template_file.read()
contents_template_object = ContentsFormat(template_text,
'/path/to/template',
ignore_comments=True)

contents_original_object.join_template(contents_template_object)

with open('./tests/templates/format/testfiles/contents_result',
'r') as result_file:
result_text = result_file.read()

assert contents_original_object.document_text == result_text

def test_if_input_documents_are_an_original_document_without_calculate_header_and_a_template_and_the_add_header_flag_is_set__the_format_object_joins_an_original_document_and_a_template_and_adds_the_calculate_header(self):
original_text = '''
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
'''
test-category/test-package /etc/dir_2

template_text = '''
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
!sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
test-category/test-package /etc/dir_3/link_0
'''
result_contents = '''dir /etc
dir /etc/dir_0
obj /etc/dir_0/file_0 c585be6f171462940b44af994a54040d 1593525253
obj /etc/dir_0/file_1 b4ff15ea7028891a83392b490d676622 1601987083
dir /etc/dir_1
obj /etc/dir_1/file_0 b4ff15ea7028891a83392b490d676622 1601982062
dir /etc/dir_2
dir /etc/dir_3
sym /etc/dir_3/link_0 -> ../dir_1/file_0 1601991426
'''

join_result = '''#-------------------------------------------------------------------------------
# Modified by Calculate Utilities 4.0
# Processed template files:
# /path/to/template
#-------------------------------------------------------------------------------
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
dir /usr/share/bash-completion/completions
'''
contents_format = ContentsFormat(template_text, '/path/to/template')
contents_format.execute_format('target/path', chroot_path=chroot_path)

original_object = ContentsFormat(original_text, '/path/to/template',
add_header=True)
template_object = ContentsFormat(template_text, '/path/to/template')
original_object.join_template(template_object)
assert original_object.document_text == join_result

def test_if_input_documents_are_an_original_document_with_calculate_header_and_a_template_the_add_header_flag_is_set_and_the_already_changed_flag_is_not_set__the_format_object_removes_calculate_header_from_original_document__joins_an_original_document_and_a_template_and_adds_the_calculate_header(self):
original_text = '''#-------------------------------------------------------------------------------
# Modified by Calculate Utilities 4.0
# Processed template files:
# /path/to/ancient/template
#-------------------------------------------------------------------------------
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
'''
test_package = Package(test_package_name,
chroot_path=chroot_path)

if contents_format._errors:
for error in contents_format._errors:
print(error)
assert False

assert '/etc/dir_0/file_0' in test_package
assert '/etc/dir_0/file_1' in test_package

assert '/etc/dir_1' in test_package
assert '/etc/dir_1/file_0' in test_package

assert '/etc/dir_2' in test_package

assert '/etc/dir_3' in test_package
assert '/etc/dir_3/link_0' in test_package

with open(os.path.join(
chroot_path,
'var/db/pkg/test-category/test-package-1.0/CONTENTS'),
'r') as test_contents:
assert test_contents.read() == result_contents

def test_if_template_of_the_contents_format_contains_some_files_to_remove_directory_to_remove_and_link_to_remove_and_some_files_is_listed_using_glob_symbols__all_files_links_and_directories_are_removed_from_the_target_package(self):
chroot_path = os.path.join(TESTFILES_PATH, 'root_1')
test_package_name = PackageAtomName(
{'pkg_path':
os.path.join(
chroot_path,
'var/db/pkg/test-category/test-package-1.0'),
'version': Version('1.0')})
template_text = '''
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
!sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
'''
!test-category/test-package /etc/dir_0/file_1

join_result = '''#-------------------------------------------------------------------------------
# Modified by Calculate Utilities 4.0
# Processed template files:
# /path/to/template
#-------------------------------------------------------------------------------
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
dir /usr/share/bash-completion/completions
'''
!test-category/test-package /etc/dir_1/*

!test-category/test-package /etc/dir_2/

original_object = ContentsFormat(original_text, '/path/to/template',
add_header=True)
template_object = ContentsFormat(template_text, '/path/to/template')
original_object.join_template(template_object)
assert original_object.document_text == join_result

def test_if_input_documents_are_an_original_document_with_calculate_header_and_a_template_the_add_header_flag_is_set_and_the_already_changed_flag_is_set__the_format_object_joins_an_original_document_and_a_template_and_adds_the_calculate_header_with_a_template_paths_from_the_old_header_and_paths_to_a_current_template(self):
original_text = '''#-------------------------------------------------------------------------------
# Modified by Calculate Utilities 4.0
# Processed template files:
# /path/to/ancient/template
#-------------------------------------------------------------------------------
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
!test-category/test-package /etc/dir_3/link_0
'''
result_contents = '''dir /etc
dir /etc/dir_0
obj /etc/dir_0/file_0 c585be6f171462940b44af994a54040d 1593525253
dir /etc/dir_1
dir /etc/dir_3
'''

contents_format = ContentsFormat(template_text, '/path/to/template')
contents_format.execute_format('target/path', chroot_path=chroot_path)

test_package = Package(test_package_name,
chroot_path=chroot_path)

if contents_format._errors:
for error in contents_format._errors:
print(error)
assert False

assert '/etc/dir_0/file_0' in test_package
assert '/etc/dir_0/file_1' not in test_package

assert '/etc/dir_1' in test_package
assert '/etc/dir_1/file_0' not in test_package
assert '/etc/dir_1/file_1' not in test_package

assert '/etc/dir_3' in test_package
assert '/etc/dir_3/link_0' not in test_package

with open(os.path.join(
chroot_path,
'var/db/pkg/test-category/test-package-1.0/CONTENTS'),
'r') as test_contents:
assert test_contents.read() == result_contents

def test_if_template_of_the_contents_format_contains_some_files_to_move_from_a_source_package_to_a_target_package_directory_to_move_and_link_to_move_and_some_files_is_listed_using_glob_symbols__all_files_links_and_directories_are_moved_from_the_source_package_to_the_target_package(self):
chroot_path = os.path.join(TESTFILES_PATH, 'root_2')
test_package_name = PackageAtomName(
{'pkg_path':
os.path.join(
chroot_path,
'var/db/pkg/test-category/test-package-1.0'),
'version': Version('1.0')})
other_package_name = PackageAtomName(
{'pkg_path':
os.path.join(
chroot_path,
'var/db/pkg/test-category/other-package-1.1'),
'version': Version('1.1')})
other_package = Package(other_package_name,
chroot_path=chroot_path)
template_text = '''
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
!sym /usr/share/bash-completion/completions/ex -> vim 1573538054
dir /usr/share/bash-completion/completions
'''
test-category/test-package, test-category/other-package /etc/dir_0/file_0
test-category/other-package, test-category/test-package /etc/dir_0/file_1

test-category/test-package, test-category/other-package /etc/dir_1/file_[1,3]
test-category/other-package, test-category/test-package /etc/dir_1/file_[0,2]

join_result = '''#-------------------------------------------------------------------------------
# Modified by Calculate Utilities 4.0
# Processed template files:
# /path/to/ancient/template
# /path/to/template
#-------------------------------------------------------------------------------
obj /usr/bin/vim 30acc0f256e11c1ecdb1bd80b688d238 1573538056
dir /usr/share/bash-completion/completions
test-category/test-package, test-category/other-package /etc/dir_3/
test-category/other-package, test-category/test-package /etc/dir_4/

test-category/test-package, test-category/other-package /etc/dir_5/link_0
test-category/other-package, test-category/test-package /etc/dir_5/link_1
'''
test_result = '''dir /etc
dir /etc/dir_0
obj /etc/dir_0/file_1 c585be6f171462940b44af994a54040d 1593525253
dir /etc/dir_1
obj /etc/dir_1/file_0 b4ff15ea7028891a83392b490d676622 1601987083
obj /etc/dir_1/file_2 b4ff15ea7028891a83392b490d676622 1601987083
obj /etc/dir_1/file_4 b4ff15ea7028891a83392b490d676622 1601987083
dir /etc/dir_4
obj /etc/dir_4/file_0 b4ff15ea7028891a83392b490d676622 1601982062
dir /etc/dir_5
sym /etc/dir_5/link_1 -> ../dir_1/file_0 1601991426
'''
other_result = '''dir /etc
dir /etc/dir_0
obj /etc/dir_0/file_0 c585be6f171462940b44af994a54040d 1593525253
dir /etc/dir_1
obj /etc/dir_1/file_1 b4ff15ea7028891a83392b490d676622 1601987083
obj /etc/dir_1/file_3 b4ff15ea7028891a83392b490d676622 1601987083
obj /etc/dir_1/file_5 b4ff15ea7028891a83392b490d676622 1601987083
dir /etc/dir_3
obj /etc/dir_3/file_0 b4ff15ea7028891a83392b490d676622 1601982062
dir /etc/dir_5
sym /etc/dir_5/link_0 -> ../dir_1/file_0 1601991426
'''

original_object = ContentsFormat(original_text, '/path/to/template',
add_header=True, already_changed=True)
template_object = ContentsFormat(template_text, '/path/to/template')
original_object.join_template(template_object)
assert original_object.document_text == join_result
contents_format = ContentsFormat(template_text, '/path/to/template')
contents_format.execute_format('target/path', chroot_path=chroot_path)

test_package = Package(test_package_name,
chroot_path=chroot_path)
other_package = Package(other_package_name,
chroot_path=chroot_path)

assert '/etc/dir_0/file_0' not in test_package
assert '/etc/dir_0/file_1' not in other_package
assert '/etc/dir_0/file_1' in test_package
assert '/etc/dir_0/file_0' in other_package

assert '/etc/dir_1/file_1' not in test_package
assert '/etc/dir_1/file_3' not in test_package
assert '/etc/dir_1/file_0' not in other_package
assert '/etc/dir_1/file_2' not in other_package

assert '/etc/dir_1/file_0' in test_package
assert '/etc/dir_1/file_2' in test_package
assert '/etc/dir_1/file_4' in test_package
assert '/etc/dir_1/file_1' in other_package
assert '/etc/dir_1/file_3' in other_package
assert '/etc/dir_1/file_5' in other_package

assert '/etc/dir_3' not in test_package
assert '/etc/dir_3/file_0' not in test_package
assert '/etc/dir_4' not in other_package
assert '/etc/dir_4/file_0' not in other_package
assert '/etc/dir_4' in test_package
assert '/etc/dir_4/file_0' in test_package
assert '/etc/dir_3' in other_package
assert '/etc/dir_3/file_0' in other_package

assert '/etc/dir_5/link_0' not in test_package
assert '/etc/dir_5/link_1' not in other_package
assert '/etc/dir_5/link_1' in test_package
assert '/etc/dir_5/link_0' in other_package

with open(test_package_name.contents_path, 'r') as test_contents:
assert test_contents.read() == test_result

with open(other_package_name.contents_path,
'r') as other_contents:
assert other_contents.read() == other_result

def test_remove_testfiles(self):
entries = os.scandir('./tests/templates/format/testfiles/contents')
for root_entry in entries:
if root_entry.path.endswith('.backup'):
shutil.rmtree(root_entry.path.rstrip('.backup'))

+ 3
- 3
tests/templates/format/test_dovecot.py View File

@@ -239,12 +239,12 @@ class TestParsingMethods:
assert dovecot_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/dovecot_original.conf', 'r') as original_file:
with open('./tests/templates/format/testfiles/dovecot/original.conf', 'r') as original_file:
original_text = original_file.read()
dovecot_original_object = DovecotFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/dovecot_template.conf', 'r') as template_file:
with open('./tests/templates/format/testfiles/dovecot/template.conf', 'r') as template_file:
template_text = template_file.read()
dovecot_template_object = DovecotFormat(template_text,
'/path/to/template',
@@ -252,7 +252,7 @@ class TestParsingMethods:

dovecot_original_object.join_template(dovecot_template_object)

with open('./tests/templates/format/testfiles/dovecot_result.conf', 'r') as result_file:
with open('./tests/templates/format/testfiles/dovecot/result.conf', 'r') as result_file:
result_text = result_file.read()

assert dovecot_original_object.document_text == result_text


+ 6
- 3
tests/templates/format/test_json.py View File

@@ -28,17 +28,20 @@ class TestParsingMethods:
assert jsonObject._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/json_original.json', 'r') as originalFile:
with open('./tests/templates/format/testfiles/json/'
'original.json', 'r') as originalFile:
originalText = originalFile.read()
jsonOriginalObject = JSONFormat(originalText, '/path/to/template')

with open('./tests/templates/format/testfiles/json_template.json', 'r') as templateFile:
with open('./tests/templates/format/testfiles/json/'
'template.json', 'r') as templateFile:
templateText = templateFile.read()
jsonTemplateObject = JSONFormat(templateText, '/path/to/template')

jsonOriginalObject.join_template(jsonTemplateObject)

with open('./tests/templates/format/testfiles/json_result.json', 'r') as resultFile:
with open('./tests/templates/format/testfiles/json/'
'result.json', 'r') as resultFile:
resultText = resultFile.read()

assert jsonOriginalObject.document_text == resultText


+ 4
- 4
tests/templates/format/test_kde.py View File

@@ -221,12 +221,12 @@ class TestParsingMethods:
assert kde_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/kde_original',
with open('./tests/templates/format/testfiles/kde/original',
'r') as original_file:
original_text = original_file.read()
kde_original_object = KDEFormat(original_text, '/path/to/template')

with open('./tests/templates/format/testfiles/kde_template',
with open('./tests/templates/format/testfiles/kde/template',
'r') as template_file:
template_text = template_file.read()
kde_template_object = KDEFormat(template_text, '/path/to/template',
@@ -234,7 +234,7 @@ class TestParsingMethods:

kde_original_object.join_template(kde_template_object)

with open('./tests/templates/format/testfiles/kde_result',
with open('./tests/templates/format/testfiles/kde/result',
'r') as result_file:
result_text = result_file.read()

@@ -384,7 +384,7 @@ pluginWhiteList=shell,bookmarks,locations
'''

original_object = KDEFormat(original_text, '/path/to/template',
add_header=True, already_changed=True)
add_header=True, already_changed=True)
template_object = KDEFormat(template_text, '/path/to/template')
original_object.join_template(template_object)
assert original_object.document_text == join_result

+ 3
- 3
tests/templates/format/test_kernel.py View File

@@ -95,13 +95,13 @@ class TestParsingMethods:
assert kernel_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/kernel_original',
with open('./tests/templates/format/testfiles/kernel/original',
'r') as original_file:
original_text = original_file.read()
kernel_original_object = KernelFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/kernel_template',
with open('./tests/templates/format/testfiles/kernel/template',
'r') as template_file:
template_text = template_file.read()
kernel_template_object = KernelFormat(template_text,
@@ -110,7 +110,7 @@ class TestParsingMethods:

kernel_original_object.join_template(kernel_template_object)

with open('./tests/templates/format/testfiles/kernel_result',
with open('./tests/templates/format/testfiles/kernel/result',
'r') as result_file:
result_text = result_file.read()



+ 4
- 4
tests/templates/format/test_ldap.py View File

@@ -7,7 +7,7 @@ from calculate.templates.format.ldap_format import LDAPFormat
class TestParsingMethods:
def test_if_logiclines_method_takes_text_with_lines_that_starts_whit_space_symbols__it_returns_joined_lines(self):
with open(
'./tests/templates/format/testfiles/ldap_logic_lines_test.txt',
'./tests/templates/format/testfiles/ldap/logic_lines_test.txt',
'r') as input_file:
input_text = input_file.read()

@@ -357,13 +357,13 @@ database bdb
assert ldap_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/ldap_original.conf',
with open('./tests/templates/format/testfiles/ldap/original.conf',
'r') as original_file:
original_text = original_file.read()
ldap_original_object = LDAPFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/ldap_template.conf',
with open('./tests/templates/format/testfiles/ldap/template.conf',
'r') as template_file:
template_text = template_file.read()
ldap_template_object = LDAPFormat(template_text,
@@ -372,7 +372,7 @@ database bdb

ldap_original_object.join_template(ldap_template_object)

with open('./tests/templates/format/testfiles/ldap_result.conf',
with open('./tests/templates/format/testfiles/ldap/result.conf',
'r') as result_file:
result_text = result_file.read()



+ 3
- 3
tests/templates/format/test_openrc.py View File

@@ -131,13 +131,13 @@ class TestParsingMethods:
assert openrc_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/openrc_original',
with open('./tests/templates/format/testfiles/openrc/original',
'r') as original_file:
original_text = original_file.read()
openrc_original_object = OpenRCFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/openrc_template',
with open('./tests/templates/format/testfiles/openrc/template',
'r') as template_file:
template_text = template_file.read()
openrc_template_object = OpenRCFormat(template_text,
@@ -146,7 +146,7 @@ class TestParsingMethods:

openrc_original_object.join_template(openrc_template_object)

with open('./tests/templates/format/testfiles/openrc_result',
with open('./tests/templates/format/testfiles/openrc/result',
'r') as result_file:
result_text = result_file.read()



+ 15
- 17
tests/templates/format/test_patch.py View File

@@ -7,33 +7,31 @@ import os
import re


BACKUP_PATH = os.path.join(
os.getcwd(),
'tests/templates/format/testfiles/patch_testfiles.backup')
BACKUP_PATH = os.path.join(os.getcwd(),
'tests/templates/format/'
'testfiles/patch/root.backup')

TESTFILES_PATH = os.path.join(
os.getcwd(),
'tests/templates/format/testfiles/patch_testfiles')
TEST_ROOT_PATH = os.path.join(os.getcwd(),
'tests/templates/format/testfiles/patch/root')


@pytest.mark.patch
class TestExecuteMethods:
def test_create_testfiles(self):
shutil.copytree(BACKUP_PATH, TESTFILES_PATH)
shutil.copytree(BACKUP_PATH, TEST_ROOT_PATH)

def test_if_diff_patch_used_for_patching_of_several_files__it_changes_patched_file_correctly(self):
if not os.path.exists(TESTFILES_PATH):
if not os.path.exists(TEST_ROOT_PATH):
assert False

path_pattern = re.compile(r'/a1/')
test_result = True
cwd_path = path.join(
os.getcwd(),
'tests/templates/format/testfiles/patch_testfiles')
cwd_path = path.join(os.getcwd(),
'tests/templates/format/testfiles/patch/root')
with open(path.join(cwd_path, 'diff_1.patch')) as patch_file:
patch_text = patch_file.read()

diff_patch = PatchFormat(patch_text)
diff_patch = PatchFormat(patch_text, '/template/path')
output = diff_patch.execute_format(target_path=cwd_path)
if output:
for changed_file, change_type in diff_patch.changed_files.items():
@@ -50,21 +48,21 @@ class TestExecuteMethods:
assert test_result

def test_if_diff_patch_used_for_patching_of_directories__it_changes_files_in_directories_and_adds_ones(self):
if not os.path.exists(TESTFILES_PATH):
if not os.path.exists(TEST_ROOT_PATH):
assert False

path_pattern = re.compile(r'/a1/')
test_result = True
cwd_path = path.join(
os.getcwd(),
'tests/templates/format/testfiles/patch_testfiles/a1')
'tests/templates/format/testfiles/patch/root/a1')
patch_path = path.join(
os.getcwd(),
'tests/templates/format/testfiles/patch_testfiles/diff_2.patch')
'tests/templates/format/testfiles/patch/root/diff_2.patch')
with open(path.join(patch_path)) as patch_file:
patch_text = patch_file.read()

diff_patch = PatchFormat(patch_text)
diff_patch = PatchFormat(patch_text, '/template/path')
diff_patch.execute_format(target_path=cwd_path)

for changed_file, change_type in diff_patch.changed_files.items():
@@ -79,4 +77,4 @@ class TestExecuteMethods:
assert test_result

def test_remove_testfiles(self):
shutil.rmtree(TESTFILES_PATH)
shutil.rmtree(TEST_ROOT_PATH)

+ 7
- 5
tests/templates/format/test_postfix.py View File

@@ -108,20 +108,22 @@ class TestParsingMethods:
assert postfix_object._document_dictionary == result

def test_joining_documents_1(self):
with open('./tests/templates/format/testfiles/postfix_original',
with open('./tests/templates/format/testfiles/postfix/original',
'r') as original_file:
original_text = original_file.read()
postfix_original_object = PostfixFormat(original_text, '/path/to/template')
postfix_original_object = PostfixFormat(original_text,
'/path/to/template')

with open('./tests/templates/format/testfiles/postfix_template',
with open('./tests/templates/format/testfiles/postfix/template',
'r') as template_file:
template_text = template_file.read()
postfix_template_object = PostfixFormat(template_text, '/path/to/template',
postfix_template_object = PostfixFormat(template_text,
'/path/to/template',
ignore_comments=True)

postfix_original_object.join_template(postfix_template_object)

with open('./tests/templates/format/testfiles/postfix_result',
with open('./tests/templates/format/testfiles/postfix/result',