Now 'merge' parameter can contain packages that does not exist is db/pkg. fixed #61

calculate/templates/template_engine.py

@@ -126,12 +126,12 @@ class Variables(MutableMapping):
 
 class ParametersProcessor:
     '''Класс для проверки и разбора параметров шаблона.'''
-    available_parameters: set  = {'name', 'path', 'append', 'chmod', 'chown',
+    available_parameters: set = {'name', 'path', 'append', 'chmod', 'chown',
-                                  'autoupdate', 'env', 'force', 'source',
+                                 'autoupdate', 'env', 'force', 'source',
-                                  'format', 'unbound', 'mirror', 'run', 'exec',
+                                 'format', 'unbound', 'mirror', 'run', 'exec',
-                                  'env', 'package', 'merge', 'postmerge',
+                                 'env', 'package', 'merge', 'postmerge',
-                                  'action', 'rebuild', 'restart', 'stop',
+                                 'action', 'rebuild', 'restart', 'stop',
-                                  'start', 'handler', 'notify', 'group'}
+                                 'start', 'handler', 'notify', 'group'}
 
     inheritable_parameters: set = {'chmod', 'chown', 'autoupdate', 'env',
                                    'package', 'action', 'handler', 'group'}
@@ -374,9 +374,13 @@ class ParametersProcessor:
 
         for package_name in packages_names:
             package_name = package_name.strip()
-            atom_object = self.package_atom_parser.parse_package_parameter(
+            try:
-                                                                  package_name)
+                atom_object = self.package_atom_parser.\
-            packages_list.append(atom_object)
+                                       parse_package_parameter(package_name)
+                packages_list.append(atom_object)
+            except PackageAtomError as error:
+                print("NOT FOUND", str(error))
+                continue
 
         return packages_list
 
@@ -625,8 +629,6 @@ class ParametersProcessor:
                                      " in handler templates.")
 
     def check_postparse_package(self, parameter_value):
-        print('check package value:', parameter_value)
-        print('groups:', self._groups)
         groups = []
         package_atom = PackageAtomParser.parse_atom_name(parameter_value)
 
@@ -650,7 +652,6 @@ class ParametersProcessor:
                         raise IncorrectParameter(error.message)
             elif self._check_package_group(package_atom,
                                            self._groups[group]):
-                print('successfully checked')
                 if (self._parameters_container is not None
                    and self._parameters_container.package):
                     self._parameters_container.remove_parameter('package')
@@ -663,11 +664,8 @@ class ParametersProcessor:
     def _check_package_group(self, package: dict, group_packages: list):
         '''Метод для проверки соответствия описания пакета, заданного словарем,
         какому-либо описанию пакета, заданного в переменных groups.'''
-        print(f'CHECK\n{package}')
         for group_package in group_packages:
-            print(f'WITH GROUP PACKAGE {group_package}')
             for parameter in ['category', 'name', 'version', 'slot']:
-                print(f"checkin' {parameter}")
                 if package[parameter] is not None:
                     if (group_package[parameter] is None
                        or group_package[parameter] != package[parameter]):

calculate/templates/template_processor.py

@@ -2095,9 +2095,6 @@ class DirectoryProcessor:
                         groups_namespace[namespace] = Variables()
                         groups_namespace = groups_namespace[namespace]
 
-        print('GROUPS NAMESPACE:')
-        print(groups_namespace)
-
         atom_dict = PackageAtomParser.parse_atom_name(package_atom)
         if isinstance(self.datavars_module, (Datavars, NamespaceNode)):
             if group_name not in groups_namespace:

calculate/utils/package.py

@@ -4,8 +4,13 @@ import os
 import re
 import glob
 from collections import OrderedDict
-from typing import Union
 from .files import read_file, read_link, join_paths, FilesError
+from typing import (
+    Generator,
+    Union,
+    List,
+    Any
+    )
 from pyparsing import (
     Literal,
     Regex,
@@ -32,7 +37,8 @@ DEFAULT, NOTEXIST, NOTCORRECT = range(3)
 
 class PackageAtomError(Exception):
     '''Исключение выбрасываемое при ошибках разбора ATOM-названий.'''
-    def __init__(self, message='Package atom error', errno=DEFAULT):
+    def __init__(self, message: str = 'Package atom error',
+                 errno: int = DEFAULT):
         self.message = message
         self.errno = errno
 
@@ -151,7 +157,8 @@ class Version:
 
         return version_value
 
-    def _compare_lists(self, lversion, rversion, filler=0):
+    def _compare_lists(self, lversion: list, rversion: list, filler: Any = 0
+                       ) -> int:
         '''Метод для сравнения двух списков, даже если если они не одинаковы
         по длине. Возвращает 0, если списки равны, 1 если lversion больше, -1
         если lversion меньше.'''
@@ -323,7 +330,7 @@ class ContentsParser(metaclass=Singleton):
         value = ({path: result_dict})
         return value
 
-    def parse(self, contents_text: str):
+    def parse(self, contents_text: str) -> OrderedDict:
         output_dictionary = OrderedDict()
         for tokens, start, end in self._parser.scanString(contents_text):
             parse_result = tokens[0]
@@ -332,7 +339,7 @@ class ContentsParser(metaclass=Singleton):
             output_dictionary.update(parse_result)
         return output_dictionary
 
-    def render(self, contents_dictionary: dict):
+    def render(self, contents_dictionary: dict) -> str:
         file_loader = PackageLoader('calculate', 'utils')
         environment = Environment(loader=file_loader,
                                   trim_blocks=True,
@@ -345,7 +352,7 @@ class ContentsParser(metaclass=Singleton):
 class PackageAtomName:
     '''Класс для хранения результата определения пакета. Для определения пакета
     использует путь к его pkg директории.'''
-    def __init__(self, atom_dictionary):
+    def __init__(self, atom_dictionary: dict):
         self._package_directory = atom_dictionary['pkg_path']
         self._version = atom_dictionary['version']
         if self._package_directory is not None:
@@ -421,13 +428,13 @@ class PackageAtomName:
     def slot_specified(self) -> bool:
         return self._with_slot
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other: Any) -> bool:
         if isinstance(other, PackageAtomName):
             return self._package_directory == other._package_directory
         else:
             return False
 
-    def __ne__(self, other) -> bool:
+    def __ne__(self, other: Any) -> bool:
         if isinstance(other, PackageAtomName):
             return self._package_directory != other._package_directory
         else:
@@ -456,19 +463,23 @@ class PackageAtomName:
 NonePackage = PackageAtomName({'pkg_path': None, 'version': None})
 
 
-class PackageAtomParser:
+def make_version_pattern() -> str:
-    '''Класс для парсинга параметра package, его проверки, а также определения
-    принадлежности файла пакету.'''
     _value = r'(?P<value>\d+(\.\d+)*)'
     _literal = r'(?P<literal>[a-z])?'
     _suffix = r'(?P<suffix>(_(pre|p|beta|alpha|rc)(\d+)?)+)?'
     _revision = r'(?P<revision>-r\d+)?'
-    _version_pattern = _value + _literal + _suffix + _revision
+    return _value + _literal + _suffix + _revision
+
 
-    package_name_pattern =\
+class PackageAtomParser:
+    '''Класс для парсинга параметра package, его проверки, а также определения
+    принадлежности файла пакету.'''
+    _version_pattern: str = make_version_pattern()
+
+    package_name_pattern: str =\
         fr'(?P<name>\D[\w\d]*(\-\D[\w\d]*)*)(?P<version>-{_version_pattern})?'
 
-    atom_name_pattern = r'''(?P<condition>[=~><])?
+    atom_name_pattern: str = r'''(?P<condition>[=~><])?
                             (?P<category>[^\s/]*)/
                             {0}
                             (?P<slot>:[^\[\s]*)?
@@ -479,19 +490,20 @@ class PackageAtomParser:
     package_name_regex = re.compile(package_name_pattern)
     version_regex = re.compile(_version_pattern)
 
-    atom_dict_fields = ['category', 'name', 'version', 'slot', 'use_flags',
+    atom_dict_fields: List[str] = ['category', 'name', 'version', 'slot',
-                        'with_slot', 'condition']
+                                   'use_flags', 'with_slot', 'condition']
 
-    def __init__(self, pkg_path='/var/db/pkg',
+    def __init__(self, pkg_path: str = '/var/db/pkg',
-                 chroot_path='/'):
+                 chroot_path: str = '/'):
-        self.chroot_path = chroot_path
+        self.chroot_path: str = chroot_path
 
+        self.pkg_path: str
         if chroot_path != '/':
             self.pkg_path = join_paths(chroot_path, pkg_path)
         else:
             self.pkg_path = pkg_path
 
-        self.package_atom = ''
+        self.package_atom: str = ''
 
     def parse_package_parameter(self, package_atom: Union[str, dict]
                                 ) -> PackageAtomName:
@@ -533,8 +545,7 @@ class PackageAtomParser:
 
         if not glob_result:
             # Если ничего не нашлось.
-            raise PackageAtomError("Package from 'package' parameter value"
+            raise PackageAtomError("Package '{}' is not found.".format(
-                                   " '{}' does not exist".format(
                                               atom_dictionary['package_atom']),
                                    errno=NOTEXIST)
 
@@ -605,8 +616,7 @@ class PackageAtomParser:
                                         atom_dictionary['package_atom'])
 
             if slot != atom_dictionary['slot']:
-                raise PackageAtomError("Package from 'package' parameter value"
+                raise PackageAtomError("Package '{}' is not found.".format(
-                                       " '{}' does not exist".format(
                                               atom_dictionary['package_atom']),
                                        errno=NOTEXIST)
 
@@ -625,11 +635,9 @@ class PackageAtomParser:
                         continue
                 elif use_flag in use_flags:
                     continue
-                raise PackageAtomError(
+                raise PackageAtomError("Package '{}' is not found".format(
-                                   "Package from 'package' parameter value"
+                                              atom_dictionary['package_atom']),
-                                   " '{}' does not exist".format(
+                                       errno=NOTEXIST)
-                                          atom_dictionary['package_atom']),
-                                   errno=NOTEXIST)
 
     def _get_slot_value(self, pkg_path: str, package_atom: str) -> str:
         '''Метод для получения значения slot из файла SLOT.'''
@@ -646,8 +654,8 @@ class PackageAtomParser:
         try:
             return read_file(use_path).strip('\n').split(' ')
         except FilesError:
-            raise PackageAtomError("could not read use flags for 'package'"
+            raise PackageAtomError("could not read use flags in atom name: {}".
-                                   " parameter: {}".format(package_atom))
+                                   format(package_atom))
 
     def _get_category_packages(self, category: str) -> str:
         '''Генератор имен категорий, имеющихся в /var/db/pkg'''
@@ -655,7 +663,8 @@ class PackageAtomParser:
                                                           category)):
             yield path
 
-    def _check_version(self, atom_dictionary: dict, pkg_version: Version):
+    def _check_version(self, atom_dictionary: dict, pkg_version: Version
+                       ) -> None:
         condition = atom_dictionary['condition']
 
         if condition == '=':
@@ -675,11 +684,9 @@ class PackageAtomParser:
             condition_result = False
 
         if not condition_result:
-            raise PackageAtomError(
+            raise PackageAtomError("Package '{}' is not found".format(
-                      "Package from 'package' parameter value"
+                                              atom_dictionary['package_atom']),
-                      " '{}' does not exist".format(
+                                   errno=NOTEXIST)
-                                      atom_dictionary['package_atom']),
-                      errno=NOTEXIST)
 
     def get_file_package(self, file_path: str) -> PackageAtomName:
         '''Метод для определения пакета, которому принадлежит файл.'''
@@ -718,7 +725,7 @@ class PackageAtomParser:
         if (not parsing_result or parsing_result.string != atom_name or
            not parsing_result.groupdict()['category'] or
            not parsing_result.groupdict()['name']):
-            raise PackageAtomError("'package' parameter value '{}' is not"
+            raise PackageAtomError("atom name '{}' is not"
                                    " correct".format(atom_name),
                                    errno=NOTCORRECT)
 
@@ -734,9 +741,8 @@ class PackageAtomParser:
 
         if parsing_result['condition'] is not None:
             if parsing_result['version'] is None:
-                raise PackageAtomError("'package' parameter value"
+                raise PackageAtomError(f"Atom name '{atom_name}' is not"
-                                       f" '{atom_name}' is not correct."
+                                       " correct. Version value is missed",
-                                       " Version value is missed",
                                        errno=NOTCORRECT)
         elif parsing_result['version'] is not None:
             parsing_result['condition'] = '='
@@ -768,8 +774,10 @@ class Package:
     '''Класс для работы с принадлежностью файлов пакетам.'''
     re_cfg = re.compile(r'/\._cfg\d{4}_')
 
-    def __init__(self, package_atom, pkg_path='/var/db/pkg', chroot_path='/'):
+    def __init__(self, package_atom: Union[str, PackageAtomName],
-        self.chroot_path = chroot_path
+                 pkg_path: str = '/var/db/pkg',
+                 chroot_path: str = '/'):
+        self.chroot_path: str = chroot_path
 
         self.contents_file_path = self._get_contents_path(package_atom)
         self.package_name = package_atom
@@ -786,7 +794,8 @@ class Package:
         self.contents_dictionary = OrderedDict()
         self.read_contents_file()
 
-    def _get_contents_path(self, package_atom):
+    def _get_contents_path(self, package_atom: Union[str, PackageAtomName]
+                           ) -> str:
         '''Метод для получения из ATOM-названия или готового объекта
         PackageAtomName пути к файлу CONTENTS.'''
         if isinstance(package_atom, str):
@@ -804,11 +813,11 @@ class Package:
                            "Incorrect 'package_atom' value: '{}', type: '{}''".
                            format(package_atom, type(package_atom)))
 
-    def remove_cfg_prefix(self, file_name):
+    def remove_cfg_prefix(self, file_name: str) -> str:
         '''Метод для удаления префикса ._cfg????_.'''
         return self.re_cfg.sub('/', file_name)
 
-    def remove_chroot_path(self, file_name):
+    def remove_chroot_path(self, file_name: str) -> str:
         '''Метод для удаления из пути файла корневого пути, если он не
         является /.'''
         if self.chroot_path != '/' and file_name.startswith(self.chroot_path):
@@ -853,7 +862,9 @@ class Package:
             return self.contents_dictionary[file_path]['type']
         return None
 
-    def sort_contents_dictionary(self):
+    def sort_contents_dictionary(self) -> None:
+        '''Метод для сортировки словаря, полученного в результате разбора и
+        изменения CONTENTS-файла.'''
         tree = {}
         for path in self.contents_dictionary.keys():
             path = path.strip('/').split('/')
@@ -866,18 +877,29 @@ class Package:
         sorted_contents = OrderedDict()
         for path in self._make_paths('/', tree):
             sorted_contents[path] = self.contents_dictionary[path]
+
         self.contents_dictionary = sorted_contents
 
-    def _make_paths(self, path, level):
+    # def _make_paths(self, path: str, level: dict) -> List[str]:
-        paths = []
+    #     paths = []
+    #     for part in sorted(level.keys()):
+    #         part_path = os.path.join(path, part)
+    #         paths.append(part_path)
+    #         if level[part]:
+    #             paths.extend(self._make_paths(part_path, level[part]))
+    #     return paths
+
+    def _make_paths(self, path: str,
+                    level: dict) -> Generator[str, None, None]:
+        '''Генератор используемый для преобразования дерева сортировки путей в
+        последовательность путей.'''
         for part in sorted(level.keys()):
             part_path = os.path.join(path, part)
-            paths.append(part_path)
+            yield part_path
             if level[part]:
-                paths.extend(self._make_paths(part_path, level[part]))
+                yield from self._make_paths(part_path, level[part])
-        return paths
 
-    def add_dir(self, file_name):
+    def add_dir(self, file_name: str) -> None:
         '''Метод для добавления в CONTENTS директорий.'''
         file_name = self.remove_chroot_path(file_name)
 
@@ -888,7 +910,8 @@ class Package:
             contents_item = OrderedDict({'type': 'dir'})
             self.contents_dictionary[file_name] = contents_item
 
-    def add_sym(self, file_name, target_path=None, mtime=None):
+    def add_sym(self, file_name: str, target_path: Union[str, None] = None,
+                mtime: Union[str, None] = None) -> None:
         '''Метод для добавления в CONTENTS символьных ссылок.'''
         real_path = file_name
 
@@ -913,7 +936,8 @@ class Package:
 
         self.contents_dictionary[file_name] = contents_item
 
-    def add_obj(self, file_name, file_md5=None, mtime=None):
+    def add_obj(self, file_name: str, file_md5: Union[str, None] = None,
+                mtime: Union[str, None] = None) -> None:
         '''Метод для добавления в CONTENTS обычных файлов как obj.'''
         real_path = file_name
         file_name = self.remove_chroot_path(file_name)
@@ -938,7 +962,7 @@ class Package:
                                      'mtime': mtime})
         self.contents_dictionary[file_name] = contents_item
 
-    def add_file(self, file_name):
+    def add_file(self, file_name: str) -> None:
         '''Метод для добавления в CONTENTS файла любого типа.'''
         if file_name != '/':
             real_path = file_name
@@ -954,7 +978,7 @@ class Package:
             elif os.path.isfile(real_path):
                 self.add_obj(file_name)
 
-    def remove_obj(self, file_path):
+    def remove_obj(self, file_path: str) -> OrderedDict:
         '''Метод для удаления файлов и ссылок.'''
         file_path = self.remove_chroot_path(file_path)
         file_path = self.remove_cfg_prefix(file_path)
@@ -965,7 +989,7 @@ class Package:
                             self.contents_dictionary.pop(file_path)})
         return removed
 
-    def remove_dir(self, file_path):
+    def remove_dir(self, file_path: str) -> OrderedDict:
         '''Метод для удаления из CONTENTS файлов и директорий находящихся
         внутри удаляемой директории и самой директории.'''
         directory_path = self.remove_chroot_path(file_path)
@@ -982,7 +1006,7 @@ class Package:
 
         return removed
 
-    def remove_file(self, file_path):
+    def remove_file(self, file_path: str) -> OrderedDict:
         file_path = self.remove_chroot_path(file_path)
         file_path = self.remove_cfg_prefix(file_path)
         removed = OrderedDict()
@@ -996,7 +1020,7 @@ class Package:
                             self.contents_dictionary.pop(file_path)})
         return removed
 
-    def clear_dir(self, file_path):
+    def clear_dir(self, file_path: str) -> OrderedDict:
         '''Метод для удаления из CONTENTS файлов и директорий находящихся
         внутри очищаемой директории.'''
         directory_path = self.remove_chroot_path(file_path)
@@ -1039,7 +1063,7 @@ class Package:
                             self.contents_dictionary.pop(file_path)})
         return removed
 
-    def get_md5(self, file_path):
+    def get_md5(self, file_path: str) -> str:
         '''Метод для получения md5 хэш-суммы указанного файла.'''
         try:
             file_text = read_file(file_path).encode()
@@ -1049,14 +1073,15 @@ class Package:
         file_md5 = hashlib.md5(file_text).hexdigest()
         return file_md5
 
-    def get_link_target(self, link_path):
+    def get_link_target(self, link_path: str) -> str:
         try:
             return read_link(link_path)
         except FilesError as error:
             raise PackageError(str(error))
 
-    def check_contents_data(self, file_path, file_md5=None,
+    def check_contents_data(self, file_path: str, file_md5: str = None,
-                            sym_target=None, symlink=False):
+                            sym_target: str = None, symlink: bool = False
+                            ) -> bool:
         '''Метод для проверки соответствия md5 хэш суммы файла той, что указана
         в файле CONTENTS.'''
         contents_path = file_path
@@ -1080,13 +1105,13 @@ class Package:
         else:
             return False
 
-    def __contains__(self, file_path):
+    def __contains__(self, file_path: str) -> bool:
         if self.chroot_path != "/":
             if file_path.startswith(self.chroot_path):
                 file_path = file_path[len(self.chroot_path):]
             file_path = self.remove_cfg_prefix(file_path)
         return file_path in self.contents_dictionary
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return '<Package: {}/{}>'.format(self.package_name.category,
                                          self.package_name.fullname)

tests/templates/test_directory_processor.py

@@ -772,7 +772,7 @@ class TestDirectoryProcessor:
                               join_paths(CHROOT_PATH,
                                          '/etc/dir_19/dir_0/file_0'): 'N'}
 
-    def test_if_template_s_directory_contains_some_directories_with_single_template_files_that_belongs_to_a_different_packages_and_target_files_does_not_exist_and_one_of_a_template_file_has_the_merge_parameter_with_other_packages_and_directory_processor_is_used_for_a_package__the_directory_processor_creates_one_file_using_template_with_actual_package_parameter_and_then_uses_the_packages_file_trees_to_merge_other_packages(self):
+    def test_if_template_s_directory_contains_some_directories_with_single_template_files_that_belongs_to_a_different_packages_and_target_files_does_not_exist_and_one_of_a_template_files_has_the_merge_parameter_with_other_packages_and_directory_processor_is_used_for_a_package__the_directory_processor_creates_one_file_using_template_with_actual_package_parameter_and_then_uses_the_packages_file_trees_to_merge_other_packages(self):
         datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH,
                                                          'templates_21')
         directory_processor = DirectoryProcessor('install',
@@ -1288,7 +1288,7 @@ class TestDirectoryProcessor:
                                          'etc/dir_56'): 'N',
                               join_paths(CHROOT_PATH,
                                          'etc/dir_56/file_0'): 'N'}
-        # datavars.main.cl['groups'] = Variables({})
+        datavars.main.cl['groups'] = Variables({})
 
     def test_solving_collisions_for_the_same_packages_from_different_slots(self):
         datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH,
@@ -1534,6 +1534,18 @@ class TestDirectoryProcessor:
         assert not os.path.exists(join_paths(CHROOT_PATH,
                                              '/etc/dir_72/file_0'))
 
+    def test_merge_parameter_with_unexisting_packages(self):
+        datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH,
+                                                         'templates_52')
+        directory_processor = DirectoryProcessor(
+                                           'install',
+                                           datavars_module=datavars,
+                                           install='test-category/test-package'
+                                                )
+        directory_processor.process_template_directories()
+        assert os.path.exists(join_paths(CHROOT_PATH, '/etc/dir_73/file_0'))
+        assert os.path.exists(join_paths(CHROOT_PATH, '/etc/dir_74/file_0'))
+
     def test_view_tree(self):
         list_path = join_paths(CHROOT_PATH, '/etc')
         show_tree(list_path)

tests/templates/testfiles/test_dir_processor_root/templates_52/install/.calculate_directory

@@ -0,0 +1 @@
+{% calculate action = "install", append = "skip" %}

tests/templates/testfiles/test_dir_processor_root/templates_52/install/dir_73/.calculate_directory

@@ -0,0 +1,2 @@
+{% calculate append = 'join', package = 'test-category/test-package',
+group = 'install' %}

tests/templates/testfiles/test_dir_processor_root/templates_52/install/dir_73/file_0

@@ -0,0 +1,3 @@
+{% calculate append = "join", format = "raw",
+merge = "test-category/new-package, test-category/unmerged-package" %}
+hey, hey, hey mr. hangman

tests/templates/testfiles/test_dir_processor_root/templates_52/install/dir_74/.calculate_directory

@@ -0,0 +1,2 @@
+{% calculate append = 'join', package = 'test-category/new-package',
+group = 'install' %}

tests/templates/testfiles/test_dir_processor_root/templates_52/install/dir_74/file_0

@@ -0,0 +1,2 @@
+{% calculate append = "join", format = "raw" %}
+I don't want to die in this digital ocean

tests/templates/testfiles/test_dir_processor_root/templates_52/install/dir_75/.calculate_directory

@@ -0,0 +1,2 @@
+{% calculate append = "skip", group = "uninstall",
+package = 'test-category/unmerged-package' %}