From 2009c654ab80a26cbe48d55556c2819cf614ce84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?=
 =?UTF-8?q?=D0=B8=D1=81?= <di@calculate.ru>
Date: Wed, 13 Jan 2021 14:27:49 +0300
Subject: [PATCH] Added processing of format errors. Patch format was fixed #69

---
 calculate/templates/format/base_format.py     | 48 +++++++++++--------
 calculate/templates/format/patch_format.py    |  9 ++--
 calculate/templates/template_processor.py     |  9 +++-
 tests/templates/test_directory_processor.py   | 28 ++++++++++-
 .../test_dir_processor_root/dir_38/file_0     |  1 -
 .../test_dir_processor_root/dir_40/file_0     |  1 -
 .../test_dir_processor_root/dir_42/file_0     |  1 -
 .../test_dir_processor_root/dir_7/file_0      |  9 ----
 .../etc.backup/file_18                        |  1 +
 .../testfiles/test_dir_processor_root/file_4  |  9 ----
 .../testfiles/test_dir_processor_root/file_5  |  9 ----
 .../templates_53/install/.calculate_directory |  1 +
 .../templates_53/install/patch                |  3 ++
 13 files changed, 72 insertions(+), 57 deletions(-)
 delete mode 100644 tests/templates/testfiles/test_dir_processor_root/dir_38/file_0
 delete mode 100644 tests/templates/testfiles/test_dir_processor_root/dir_40/file_0
 delete mode 100644 tests/templates/testfiles/test_dir_processor_root/dir_42/file_0
 delete mode 100644 tests/templates/testfiles/test_dir_processor_root/dir_7/file_0
 create mode 100644 tests/templates/testfiles/test_dir_processor_root/etc.backup/file_18
 delete mode 100644 tests/templates/testfiles/test_dir_processor_root/file_4
 delete mode 100644 tests/templates/testfiles/test_dir_processor_root/file_5
 create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_53/install/.calculate_directory
 create mode 100644 tests/templates/testfiles/test_dir_processor_root/templates_53/install/patch

diff --git a/calculate/templates/format/base_format.py b/calculate/templates/format/base_format.py
index eca6212..8f1853f 100644
--- a/calculate/templates/format/base_format.py
+++ b/calculate/templates/format/base_format.py
@@ -2,6 +2,7 @@
 #
 from collections import OrderedDict
 from jinja2 import Environment, PackageLoader
+from typing import Callable, List, Tuple
 from pprint import pprint
 from copy import copy
 import re
@@ -12,14 +13,16 @@ except ImportError:
 
 
 class FormatError(Exception):
-    pass
+    def __init__(self, message: str, executable: bool = False):
+        super().__init__(message)
+        self.executable: bool = executable
 
 
-class Format():
+class Format:
     FORMAT = 'none'
     CALCULATE_VERSION = None
 
-    def __init__(self, processing_methods):
+    def __init__(self, processing_methods: List[Callable]):
         self._processing_methods = processing_methods
         self._document_dictionary = OrderedDict()
         self._item_to_add = OrderedDict()
@@ -39,7 +42,7 @@ class Format():
         # для отладки.
         self._line_timer = 0
 
-    def _lines_to_dictionary(self, document_lines):
+    def _lines_to_dictionary(self, document_lines: List[str]) -> None:
         '''Основной метод для парсинга документа. Принимает список строк,
         к каждой строке применяет парсеры, определенные для некоторого формата.
         Первый парсер, которому удается разобрать строку используется для
@@ -68,7 +71,7 @@ class Format():
         if self._need_finish:
             self._finish_method()
 
-    def _parse_xml_to_dictionary(self, xml_document_text):
+    def _parse_xml_to_dictionary(self, xml_document_text: str) -> None:
         '''Метод для парсинга xml файлов.
         Файлы xml предварительно не разбиваются на строки, а разбираются с
         помощью модуля lxml. Перевод в словарь осуществляется методами формата,
@@ -76,17 +79,17 @@ class Format():
         root = fromstring(xml_document_text)
         self._document_dictionary = self._processing_methods[root.tag](root)
 
-    def print_dictionary(self):
+    def print_dictionary(self) -> None:
         '''Метод для отладки.'''
         pprint(self._document_dictionary)
 
-    def join_template(self, template):
+    def join_template(self, template: "Format"):
         '''Метод запускающий наложение шаблона.'''
         self._join(self._document_dictionary,
                    template._document_dictionary,
                    self._join_before)
 
-    def _get_list_of_logic_lines(self, text):
+    def _get_list_of_logic_lines(self, text: str) -> List[str]:
         '''Метод разбивающий документ на список логических строк -- то есть
         учитывающий при разбиении возможность разбиение одной строки на
         несколько с помощью бэкслеша. В некоторых форматах переопределен.'''
@@ -105,7 +108,8 @@ class Format():
                 lines_to_join.append(line[:-1])
         return list_of_lines
 
-    def _join(self, original, template, join_before):
+    def _join(self, original: OrderedDict,
+              template: OrderedDict, join_before: bool):
         '''Основной метод для наложения шаблонов путем объединения их словарей
         выполняемого рекурсивно.'''
         if template == OrderedDict():
@@ -226,7 +230,7 @@ class Format():
                     original[key_value] = forwarded_items[key_value]
                     original.move_to_end(key_value, last=False)
 
-    def make_template(self, template):
+    def make_template(self, template: "Format") -> "Format":
         '''Метод для запуска генерации шаблонов путем сравнения пары исходных
         файлов.'''
         full_diff, set_to_check = self.compare_dictionaries(
@@ -237,7 +241,9 @@ class Format():
         template_object._document_dictionary = full_diff
         return template_object
 
-    def compare_dictionaries(self, dict_1, dict_2):
+    def compare_dictionaries(self, dict_1: OrderedDict,
+                             dict_2: OrderedDict
+                             ) -> Tuple[OrderedDict, set]:
         '''Основной метод для генерации шаблонов путем сравнения пары исходных
         файлов. Работает рекурсивно.'''
         to_remove_dictionary = OrderedDict()
@@ -314,7 +320,7 @@ class Format():
         return full_diff, unchanged_set
 
     @property
-    def document_text(self):
+    def document_text(self) -> str:
         '''Метод для получения текста документа. Использует jinja2 для
         рендеринга документа.'''
         file_loader = PackageLoader('calculate.templates.format',
@@ -335,23 +341,24 @@ class Format():
         Переопределяется в форматах. Вызывается при self._need_finish = True'''
         pass
 
-    def _is_ready_to_update(self):
+    def _is_ready_to_update(self) -> bool:
         '''Метод для проверки флага self._ready_to_update, указывающего, что
         сформированная форматом секция документа, находящаяся в
         self._item_to_add, может быть добавлена в словарь документа.'''
         is_ready, self._ready_to_update = self._ready_to_update, False
         return is_ready
 
-    def _is_match(self):
+    def _is_match(self) -> bool:
         '''Метод для проверки флага self._is_match, указывающего что текущий
         парсер, использованный форматом, смог распарсить строку и использовать
         другие парсеры не нужно.'''
         is_match, self._match = self._match, False
         return is_match
 
-    def _get_header_and_document_text(self, input_text,
-                                      template_path,
-                                      already_changed=False):
+    def _get_header_and_document_text(self, input_text: str,
+                                      template_path: str,
+                                      already_changed: bool = False
+                                      ) -> Tuple[str, str]:
         '''Метод для создания заголовка измененного файла и удаления его из
         текста исходного файла.'''
         header_pattern = self._get_header_pattern()
@@ -369,8 +376,7 @@ class Format():
         document_text = re.sub(header_pattern, '', input_text)
         return header, document_text
 
-    def _make_header(self, template_paths: list):
-        print('making header...')
+    def _make_header(self, template_paths: list) -> str:
         if not self.comment_symbol:
             return ""
         elif self.comment_symbol in ("xml", "XML"):
@@ -388,7 +394,7 @@ class Format():
                     '{0}' + '-' * 79 + '\n').format(self.comment_symbol,
                                                     self.CALCULATE_VERSION)
 
-    def _get_header_pattern(self):
+    def _get_header_pattern(self) -> str:
         if self.comment_symbol in {"xml", "XML"}:
             return (r'<!--' + r'-' * 76 + r'\n' +
                     r'\s*Modified by Calculate Utilities [\d\w\.]*\n' +
@@ -402,5 +408,5 @@ class Format():
                     r'(?P<template_paths>({0}\s*[/\w\d\-_\.]*\n)+)' +
                     r'{0}' + r'-' * 79 + r'\n?').format(self.comment_symbol)
 
-    def __bool__(self):
+    def __bool__(self) -> bool:
         return bool(self._document_dictionary)
diff --git a/calculate/templates/format/patch_format.py b/calculate/templates/format/patch_format.py
index 89b2f1b..5e5659e 100644
--- a/calculate/templates/format/patch_format.py
+++ b/calculate/templates/format/patch_format.py
@@ -29,13 +29,13 @@ class PatchFormat(Format):
             self._cwd_path = path.dirname(self._cwd_path)
 
         if not path.exists(self._cwd_path):
-            raise FormatError('root path does not exist')
+            raise FormatError('root path does not exist', executable=True)
 
         if self._patch_text:
             self._patch_document()
             return self.changed_files
         else:
-            raise FormatError('empty patch file')
+            raise FormatError('empty patch file', executable=True)
 
     def _patch_document(self):
         '''Метод, производящий наложение патча путем запуска процесса patch.'''
@@ -57,7 +57,7 @@ class PatchFormat(Format):
             if patch_dry_run.success():
                 return ''
         else:
-            raise FormatError('correction failed')
+            raise FormatError('correction failed', executable=True)
 
         self._last_level = level
         patch_run = Process('patch', '-p{}'.format(level),
@@ -78,3 +78,6 @@ class PatchFormat(Format):
             return patch_run.read()
         else:
             return ''
+
+    def __bool__(self):
+        return bool(self._patch_text)
diff --git a/calculate/templates/template_processor.py b/calculate/templates/template_processor.py
index ed2d959..3864071 100644
--- a/calculate/templates/template_processor.py
+++ b/calculate/templates/template_processor.py
@@ -45,7 +45,7 @@ from typing import (
     Callable
     )
 from calculate.variables.loader import Datavars
-from .format.base_format import Format
+from .format.base_format import Format, FormatError
 from ..utils.io_module import IOModule
 from collections import OrderedDict, abc
 from contextlib import contextmanager
@@ -2780,6 +2780,13 @@ class DirectoryProcessor:
                                   format(str(error),
                                          template_path))
             return False
+        except FormatError as error:
+            if error.executable:
+                msg = 'Format execution error: {} Template: {}'
+            else:
+                msg = 'Format joining error: {} Template: {}'
+            self.output.set_error(msg.format(str(error), template_path))
+            return False
 
         if template_type == DIR:
             self.output.set_success('Processed directory: {}'.
diff --git a/tests/templates/test_directory_processor.py b/tests/templates/test_directory_processor.py
index 628a735..3f4d119 100644
--- a/tests/templates/test_directory_processor.py
+++ b/tests/templates/test_directory_processor.py
@@ -87,12 +87,14 @@ datavars = Variables({'install': install,
                       'custom': Variables()})
 
 
-def show_tree(dir_path, indent=0):
+def show_tree(dir_path: str, indent: int = 0,
+              files_only: bool = False) -> None:
     print('{}{}/'.format(' ' * 4 * indent, os.path.basename(dir_path)))
     indent += 1
     for node in os.scandir(dir_path):
         if node.is_dir():
-            show_tree(node.path, indent=indent)
+            if not files_only:
+                show_tree(node.path, indent=indent)
         else:
             print('{}{}'.format(' ' * 4 * indent, node.name))
 
@@ -1546,6 +1548,28 @@ class TestDirectoryProcessor:
         assert os.path.exists(join_paths(CHROOT_PATH, '/etc/dir_73/file_0'))
         assert os.path.exists(join_paths(CHROOT_PATH, '/etc/dir_74/file_0'))
 
+    def test_to_solve_bug_with_patch_template(self):
+        datavars.main['cl_template_path'] = os.path.join(CHROOT_PATH,
+                                                         'templates_53')
+        io = IOModule(save_messages=True)
+
+        directory_processor = DirectoryProcessor('install',
+                                                 datavars_module=datavars,
+                                                 output_module=io)
+        directory_processor.process_template_directories()
+        with open(join_paths(CHROOT_PATH, '/etc/file_18'), 'r') as output:
+            output_text = output.read()
+            assert output_text == 'A veeeeeeeeeeery interesting content\n'
+
+        # print("MESSAGES:")
+        # for message in io.messages:
+        #     print(f"{message[0]} -> {message[1]}")
+
+        assert io.messages[-1] ==\
+            ("error",
+             "Format execution error: correction failed Template:"
+             f" {join_paths(CHROOT_PATH, 'templates_53/install/patch')}")
+
     def test_view_tree(self):
         list_path = join_paths(CHROOT_PATH, '/etc')
         show_tree(list_path)
diff --git a/tests/templates/testfiles/test_dir_processor_root/dir_38/file_0 b/tests/templates/testfiles/test_dir_processor_root/dir_38/file_0
deleted file mode 100644
index b0ce886..0000000
--- a/tests/templates/testfiles/test_dir_processor_root/dir_38/file_0
+++ /dev/null
@@ -1 +0,0 @@
-Information number 1.
\ No newline at end of file
diff --git a/tests/templates/testfiles/test_dir_processor_root/dir_40/file_0 b/tests/templates/testfiles/test_dir_processor_root/dir_40/file_0
deleted file mode 100644
index 90fd5ac..0000000
--- a/tests/templates/testfiles/test_dir_processor_root/dir_40/file_0
+++ /dev/null
@@ -1 +0,0 @@
-Information number 2.
\ No newline at end of file
diff --git a/tests/templates/testfiles/test_dir_processor_root/dir_42/file_0 b/tests/templates/testfiles/test_dir_processor_root/dir_42/file_0
deleted file mode 100644
index 90fd5ac..0000000
--- a/tests/templates/testfiles/test_dir_processor_root/dir_42/file_0
+++ /dev/null
@@ -1 +0,0 @@
-Information number 2.
\ No newline at end of file
diff --git a/tests/templates/testfiles/test_dir_processor_root/dir_7/file_0 b/tests/templates/testfiles/test_dir_processor_root/dir_7/file_0
deleted file mode 100644
index 6d07c65..0000000
--- a/tests/templates/testfiles/test_dir_processor_root/dir_7/file_0
+++ /dev/null
@@ -1,9 +0,0 @@
-#-------------------------------------------------------------------------------
-# Modified by Calculate Utilities 4.0
-# Processed template files:
-# /var/calculate/development/calculate-utils-4-lib/tests/templates/testfiles/test_dir_processor_root/templates_10/root/etc/dir_7/file_0
-#-------------------------------------------------------------------------------
-options {
-    parameter-1 value_1;
-    parameter-2 value_2;
-};
diff --git a/tests/templates/testfiles/test_dir_processor_root/etc.backup/file_18 b/tests/templates/testfiles/test_dir_processor_root/etc.backup/file_18
new file mode 100644
index 0000000..09c05c3
--- /dev/null
+++ b/tests/templates/testfiles/test_dir_processor_root/etc.backup/file_18
@@ -0,0 +1 @@
+A veeeeeeeeeeery interesting content
diff --git a/tests/templates/testfiles/test_dir_processor_root/file_4 b/tests/templates/testfiles/test_dir_processor_root/file_4
deleted file mode 100644
index b65f02f..0000000
--- a/tests/templates/testfiles/test_dir_processor_root/file_4
+++ /dev/null
@@ -1,9 +0,0 @@
-#-------------------------------------------------------------------------------
-# Modified by Calculate Utilities 4.0
-# Processed template files:
-# /var/calculate/development/calculate-utils-4-lib/tests/templates/testfiles/test_dir_processor_root/templates_10/root/etc/file_4
-#-------------------------------------------------------------------------------
-options {
-    parameter-1 value_1;
-    parameter-2 value_2;
-};
diff --git a/tests/templates/testfiles/test_dir_processor_root/file_5 b/tests/templates/testfiles/test_dir_processor_root/file_5
deleted file mode 100644
index e14fceb..0000000
--- a/tests/templates/testfiles/test_dir_processor_root/file_5
+++ /dev/null
@@ -1,9 +0,0 @@
-#-------------------------------------------------------------------------------
-# Modified by Calculate Utilities 4.0
-# Processed template files:
-# /var/calculate/development/calculate-utils-4-lib/tests/templates/testfiles/test_dir_processor_root/templates_10/root/etc/file_5
-#-------------------------------------------------------------------------------
-options {
-    parameter-1 value_1;
-    parameter-2 value_2;
-};
diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_53/install/.calculate_directory b/tests/templates/testfiles/test_dir_processor_root/templates_53/install/.calculate_directory
new file mode 100644
index 0000000..6e74451
--- /dev/null
+++ b/tests/templates/testfiles/test_dir_processor_root/templates_53/install/.calculate_directory
@@ -0,0 +1 @@
+{% calculate action = 'install', append = 'skip' %}
diff --git a/tests/templates/testfiles/test_dir_processor_root/templates_53/install/patch b/tests/templates/testfiles/test_dir_processor_root/templates_53/install/patch
new file mode 100644
index 0000000..589ef5d
--- /dev/null
+++ b/tests/templates/testfiles/test_dir_processor_root/templates_53/install/patch
@@ -0,0 +1,3 @@
+{% calculate format = "patch" %}
+<reg>test</reg>
+<text>zxcv</text>