Improved parser of the CONTENTS files. fixed #35

calculate/templates/format/base_format.py

@@ -2,7 +2,7 @@
 #
 from collections import OrderedDict
 from jinja2 import Environment, PackageLoader
-from typing import Callable, List, Tuple
+from typing import Callable, List, Tuple, Union
 from pprint import pprint
 from copy import copy
 import re
@@ -19,30 +19,29 @@ class FormatError(Exception):
 
 
 class Format:
-    FORMAT = 'none'
+    FORMAT: str = 'none'
-    CALCULATE_VERSION = None
+    CALCULATE_VERSION: Union[str, None] = None
-
+    SHEBANG_PATTERN: str = r"^(?P<shebang>#!\s*[\w\d\/]+\n)"
-    SHEBANG_PATTERN = r"^(?P<shebang>#!\s*[\w\d\/]+\n)"
 
     def __init__(self, processing_methods: List[Callable]):
-        self._processing_methods = processing_methods
+        self._processing_methods: List[Callable] = processing_methods
-        self._document_dictionary = OrderedDict()
+        self._document_dictionary: OrderedDict = OrderedDict()
-        self._item_to_add = OrderedDict()
+        self._item_to_add: OrderedDict = OrderedDict()
 
-        self.TEMPLATES_DIRECTORY = 'templates'
+        self.TEMPLATES_DIRECTORY: str = 'templates'
 
-        self._fatal_error_flag = False
+        self._fatal_error_flag: bool = False
-        self._ready_to_update = False
+        self._ready_to_update: bool = False
-        self._match = False
+        self._match: bool = False
 
-        self._need_finish = False
+        self._need_finish: bool = False
-        self._comments_processing = False
+        self._comments_processing: bool = False
 
-        self._join_before = False
+        self._join_before: bool = False
-        self._join_before_in_areas = False
+        self._join_before_in_areas: bool = False
 
         # для отладки.
-        self._line_timer = 0
+        self._line_timer: int = 0
 
     def _lines_to_dictionary(self, document_lines: List[str]) -> None:
         '''Основной метод для парсинга документа. Принимает список строк,

calculate/utils/package.py

@@ -7,6 +7,7 @@ from collections import OrderedDict
 from .files import read_file, read_link, join_paths, FilesError
 from typing import (
     Generator,
+    Tuple,
     Union,
     List,
     Any
@@ -293,6 +294,67 @@ class Version:
 
 
 class ContentsParser(metaclass=Singleton):
+    def __init__(self):
+        self._parsers = {'dir': ContentsParser._parse_dir,
+                         'sym': ContentsParser._parse_sym,
+                         'obj': ContentsParser._parse_obj}
+
+        self._patterns = {'dir': "dir {path}",
+                          'sym': "sym {path} -> {target} {mtime}",
+                          'obj': "obj {path} {md5} {mtime}"}
+
+    def parse(self, text: str) -> OrderedDict:
+        output = OrderedDict()
+        for line in text.split('\n'):
+            line = line.strip()
+            if not line:
+                continue
+            parts = line.split()
+            path, value = self._parsers[parts[0]](parts)
+            output[path] = value
+        return output
+
+    @staticmethod
+    def _parse_dir(parts: List[str]) -> Tuple[str, dict]:
+        try:
+            return parts[1], {'type': 'dir'}
+        except Exception as error:
+            print(str(error))
+            print("parts:", parts)
+            raise
+
+    @staticmethod
+    def _parse_obj(parts: List[str]) -> Tuple[str, dict]:
+        try:
+            return parts[1], {'type': 'obj',
+                              'md5': parts[2].strip(),
+                              'mtime': parts[3].strip()}
+        except Exception as error:
+            print(str(error))
+            print("parts:", parts)
+            raise
+
+    @staticmethod
+    def _parse_sym(parts: List[str]) -> Tuple[str, dict]:
+        try:
+            return parts[1], {'type': 'sym',
+                              'target': parts[3].strip(),
+                              'mtime': parts[4].strip()}
+        except Exception as error:
+            print(str(error))
+            print("parts:", parts)
+            raise
+
+    def render(self, contents_dictionary: OrderedDict) -> str:
+        lines = []
+        for path, value in contents_dictionary.items():
+            lines.append(
+                      self._patterns[value['type']].format(path=path, **value))
+        lines.append('')
+        return "\n".join(lines)
+
+
+class OldContentsParser(metaclass=Singleton):
     def __init__(self):
         '''Метод для инициализации парсеров.'''
         sym_keyword = Literal('sym')

tests/templates/format/test_regex.py

@@ -1,5 +1,4 @@
 import pytest
-from collections import OrderedDict
 from calculate.templates.format.regex_format import RegexFormat
 from calculate.templates.template_engine import ParametersContainer
 

tests/templates/test_template_wrapper.py

@@ -1154,16 +1154,13 @@ class TestTemplateWrapper:
         template_wrapper.remove_from_contents()
         template_wrapper.save_changes()
 
-        try:
+        template_wrapper = TemplateWrapper(
-            template_wrapper = TemplateWrapper(
+                                   join_paths(CHROOT_PATH,
-                                       join_paths(CHROOT_PATH,
+                                              '/etc/dir/file.conf'),
-                                                  '/etc/dir/file.conf'),
+                                   parameters_object, FILE,
-                                       parameters_object, FILE,
+                                   '/path/to/template',
-                                       '/path/to/template',
+                                   chroot_path=CHROOT_PATH,
-                                       chroot_path=CHROOT_PATH,
+                                   config_archive_path=CONFIG_ARCHIVE_PATH)
-                                       config_archive_path=CONFIG_ARCHIVE_PATH)
-        except Exception as error:
-            pytest.fail("Unexpected exception: {}".format(str(error)))
         template_wrapper.add_to_contents()
         template_wrapper.save_changes()
 

tests/templates/testfiles/test_dir_processor_root/var.backup/db/pkg/test-category/test-package-1.0/CONTENTS

@@ -8,10 +8,10 @@ dir /etc/dir_60
 obj /etc/dir_60/file_0 c9a9459e4266ea35a612b90dc3653112 1593525253
 obj /etc/file_19 3fd436479300b04370b97f4bcfdc90f7 1592574626
 obj /etc/file_20 3fd436479300b04370b97f4bcfdc90f7 1592574626
-obj /etc/dir_76
+dir /etc/dir_76
 obj /etc/dir_76/file_0 3fd436479300b04370b97f4bcfdc90f7 1592574626
-obj /etc/dir_77
+dir /etc/dir_77
 obj /etc/dir_77/file_0 3fd436479300b04370b97f4bcfdc90f7 1592574626
-obj /etc/dir_78
+dir /etc/dir_78
 obj /etc/dir_78/file_0 3fd436479300b04370b97f4bcfdc90f7 1592574626
 obj /etc/dir_78/file_1 3fd436479300b04370b97f4bcfdc90f7 1592574626

tests/utils/test_package.py

@@ -109,6 +109,14 @@ dir /etc/test_dir_1
         print('TEXT:')
         print(output_text)
 
+        print("ORIGINAL LEN =", len(contents_text))
+        print("OUTPUT LEN =", len(output_text))
+        for lval, rval in zip(contents_text, output_text):
+            if lval != rval:
+                print(f"{lval} != {rval}")
+        else:
+            print("DONE")
+
         assert output_text == contents_text
 
     def test_if_PackageContents_object_initialized_by_existing_package__it_contains_dictionary_of_items_from_contents_file(self):
@@ -180,8 +188,10 @@ obj /etc/test_dir_2/file_2.cfg a371f4d456d471ac0ed0e8befff1cb6d {}
                                      'etc/test_dir_2/file_2.cfg')).st_mtime))
 
         contents_object.add_obj('/etc/test_dir_2/file_2.cfg')
+        print('ORIGINAL:')
+        print(result.encode())
         print('RESULT:')
-        print(contents_object.render_contents_file())
+        print(contents_object.render_contents_file().encode())
         assert contents_object.render_contents_file() == result
 
     def test_if_new_link_is_added_in_contents_file_using_add_sym_method__the_PackageContents_object_renders_the_contents_file_with_new_sym(self):
@@ -197,11 +207,13 @@ dir /etc
 dir /etc/test_dir_2
 sym /etc/test_dir_2/symlink -> file_2.cfg {}
 '''.format(int(os.lstat(os.path.join(CHROOT_PATH,
-                                     'etc/test_dir_2/symlink')).st_mtime))
+                        'etc/test_dir_2/symlink')).st_mtime))
 
         contents_object.add_sym('/etc/test_dir_2/symlink')
+        print('ORIGINAL:')
+        print(result.encode())
         print('RESULT:')
-        print(contents_object.render_contents_file())
+        print(contents_object.render_contents_file().encode())
         assert contents_object.render_contents_file() == result
 
     def test_if_the_PackageAtom_object_parsed_a_correct_package_atom_name_but_without_a_slot_and_use_flags__the_PackageAtom_object_returns_atom_name_of_package(self):