Template processing modules and some utils is added.

calculate/templates/format/base_format.py

@@ -10,6 +10,10 @@ except ImportError:
     from xml.etree.ElementTree import fromstring
 
 
+class FormatError(Exception):
+    pass
+
+
 class BaseFormat():
     def __init__(self, processing_methods):
         self._processing_methods = processing_methods

calculate/templates/format/diff_format.py

@@ -1,6 +1,7 @@
 # vim: fileencoding=utf-8
 #
 from calculate.utils.files import Process
+from calculate.templates.format.base_format import FormatError
 from os import path
 
 
@@ -14,21 +15,16 @@ class DiffFormat():
         self._changed_files_list = []
 
     def execute_format(self, root_path):
+        print
         if path.exists(root_path):
             self._root_path = root_path
         else:
-            # Какая-то обработка ошибки.
-            error_message = 'Root path does not exist.'
-            print(error_message)
-            return False
+            raise FormatError('Root path does not exist.')
 
         if self._patch_text:
             return self._patch_document()
         else:
-            # Какая-то обработка ошибки.
-            error_message = 'Empty patch file.'
-            print(error_message)
-            return False
+            raise FormatError('Empty patch file.')
 
     def _patch_document(self):
         for level in range(0, 4):
@@ -44,19 +40,19 @@ class DiffFormat():
             if patch_dry_run.success():
                 return ''
         else:
-            # Какая-то обработка ошибки.
-            error_message = 'Correction failed.'
-            print(error_message)
-            return False
+            raise FormatError('Correction failed.')
 
         self._last_level = level
         patch_run = Process('patch', '-p{}'.format(level), cwd=self._root_path)
         patch_run.write(self._patch_text)
 
         if patch_run.success():
+            print('patch run is successful...')
             for line in patch_run:
                 if line.startswith('patching file'):
                     self._changed_files_list.append(line[13:].strip())
             return patch_run.read()
         else:
+            print('patch run is no successful...')
+            print(patch_run.read_error())
             return ''

calculate/templates/template_engine.py

@@ -44,6 +44,8 @@ class Variables(MutableMapping):
 
 
 def resolve_or_missing(context, key, missing=missing, env={}):
+    '''Переопределение функции из для поиска значений переменных из jinja2.
+    Ищет переменные в datavars.'''
     datavars = context.parent['__datavars__']
     if key in context.vars:
         return context.vars[key]
@@ -58,6 +60,7 @@ def resolve_or_missing(context, key, missing=missing, env={}):
 
 
 class CalculateContext(Context):
+    '''Класс контекста позволяющий искать значения datavars и сохранять их.'''
     _env_set = set()
 
     def resolve(self, key):
@@ -80,20 +83,20 @@ class CalculateContext(Context):
                                   env=self._env_set)
 
 
+class ConditionFailed(TemplateSyntaxError):
+    pass
+
+
 class Parameters(MutableMapping):
-    '''Класс для хранения параметров и условий, взятых из шаблона, и передачи
+    '''Класс для хранения параметров, взятых из шаблона, и передачи
     их шаблонизатору.'''
-    def __init__(self, parameters_dictionary={}, condition=True):
+    def __init__(self, parameters_dictionary={}):
         self.__parameters = parameters_dictionary
-        self.__condition = condition
 
     def set_parameters(self, *args, **kwargs):
         parameters = dict(*args, **kwargs)
         self.__parameters.update(parameters)
 
-    def set_condition(self, condition):
-        self.__condition = self.__condition and condition
-
     def __getitem__(self, name):
         return self.__parameters[name]
 
@@ -110,12 +113,7 @@ class Parameters(MutableMapping):
         return len(self.__parameters)
 
     def __repr__(self):
-        return '<Parameters: {0}, condition={1}>'.format(self.__parameters,
-                                                         self.__condition)
-
-    @property
-    def condition(self):
-        return self.__condition
+        return '<Parameters: {0}>'.format(self.__parameters)
 
     @property
     def parameters(self):
@@ -123,7 +121,9 @@ class Parameters(MutableMapping):
 
 
 class CalculateExtension(Extension):
+    '''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.'''
     _parameters_set = set()
+    _datavars = Variables()
 
     def __init__(self, environment):
         self.tags = {'calculate', 'save', 'set_var'}
@@ -134,6 +134,8 @@ class CalculateExtension(Extension):
         self.parse_methods = {'calculate': self.parse_calculate,
                               'save': self.parse_save}
 
+        self.environment = environment
+
     def parse(self, parser):
         self.parser = parser
         self.stream = parser.stream
@@ -141,6 +143,8 @@ class CalculateExtension(Extension):
         return [self.parse_methods[tag_token]()]
 
     def parse_save(self):
+        '''Метод для разбора тега save, сохраняющего значение указанной
+        переменной datavars.'''
         lineno = next(self.stream).lineno
         target_file = nodes.Const('', lineno=lineno)
 
@@ -175,9 +179,10 @@ class CalculateExtension(Extension):
                                 lineno=lineno)
 
     def parse_calculate(self):
+        '''Метод для разбора тега calculate, содержащего значения параметров и
+        условия выполнения шаблона.'''
         pairs_list = []
         expect_comma_flag = False
-        conditions = nodes.Const(True)
 
         lineno = next(self.stream).lineno
 
@@ -185,17 +190,22 @@ class CalculateExtension(Extension):
             if expect_comma_flag:
                 self.stream.expect('comma')
 
-            if self.stream.current.type == 'name':
-                if (self.stream.current.value in self._parameters_set and
-                   self.stream.look().type != 'dot'):
-                    pairs_list.append(self.get_parameter_node())
-                else:
-                    conditions = nodes.And(
-                                       self.parser.parse_expression(
-                                                            with_condexpr=True
-                                                                   ),
-                                       conditions
-                                          )
+            if (self.stream.current.type == 'name'
+                and self.stream.current.value in self._parameters_set
+                    and self.stream.look().type != 'dot'
+                    and self.stream.look().type not in
+                    self.CONDITION_TOKENS_TYPES):
+                # разбираем параметр.
+                pairs_list.append(self.get_parameter_node())
+            elif (self.stream.current.type == 'name'
+                  or self.stream.current.type == 'lparen'):
+                # разбираем условие. Если условие False -- кидаем исключение.
+                condition_result = self.get_condition_result()
+                if not condition_result:
+                    raise ConditionFailed(
+                                          'Condition is failed',
+                                          lineno=self.stream.current.lineno
+                                         )
             else:
                 raise TemplateSyntaxError('Name is expected in calculate tag.',
                                           lineno=self.stream.current.lineno)
@@ -203,13 +213,47 @@ class CalculateExtension(Extension):
         dictionary_node = nodes.Dict(pairs_list)
         save_node = self.call_method('save_parameters',
                                      [dictionary_node,
-                                      conditions,
                                       nodes.ContextReference()],
                                      lineno=lineno)
         return nodes.Output([save_node], lineno=lineno)
 
+    def get_condition_result(self):
+        '''Метод для разбора условий из тега calculate.'''
+        condition_list = []
+        # собираем исходный код условия из токенов.
+        # вероятно, следует придумать лучший способ.
+        while (self.stream.current.type != 'block_end' and
+               self.stream.current.type != 'comma'):
+            if self.stream.current.type == 'string':
+                condition_list.append("'{}'".format(
+                                          self.stream.current.value
+                                                   ))
+            elif self.stream.current.type == 'dot':
+                self.stream.skip(1)
+                if self.stream.current.type == 'name':
+                    next_name = '.' + self.stream.current.value
+                else:
+                    raise TemplateSyntaxError(
+                                  'Variable name is not correct.',
+                                  lineno=self.stream.current.lineno
+                                             )
+                condition_list[-1] = condition_list[-1] + next_name
+            else:
+                condition_list.append(
+                                         str(self.stream.current.value)
+                                     )
+            self.stream.skip(1)
+
+        condition = ' '.join(condition_list)
+
+        # компилируем исходный код условия и получаем результат его вычисления.
+        cond_expr = self.environment.compile_expression(condition)
+        condition_result = cond_expr(__datavars__=self._datavars)
+        return condition_result
+
     def save_variable(self, variable_name, right_value, target_file, context):
-        '''Временный метод для сохранения значений переменных.'''
+        '''Метод для сохранения значений переменных указанных в теге save.'''
+        # временная реализация.
         datavars = context.parent['__datavars__']
         module_name = variable_name[0]
         namespaces = variable_name[1:-1]
@@ -228,6 +272,7 @@ class CalculateExtension(Extension):
         return ''
 
     def get_parameter_node(self):
+        '''Метод для разбра параметров, содержащихся в теге calculate.'''
         lineno = self.stream.current.lineno
         parameter_name = self.stream.expect('name').value
         parameter_name_node = nodes.Const(parameter_name, lineno=lineno)
@@ -235,6 +280,8 @@ class CalculateExtension(Extension):
             parameter_value = self.stream.current.value
             parameter_rvalue = self.parser.parse_expression(with_condexpr=True)
             if parameter_name == 'env':
+                # если параметр env -- обновляем множенство значений env
+                # контекста вo время парсинга.
                 env_names = parameter_value.split(',')
                 for name in env_names:
                     CalculateContext._env_set.add(name.strip())
@@ -242,18 +289,19 @@ class CalculateExtension(Extension):
             parameter_rvalue = nodes.Const(True, lineno=lineno)
         return nodes.Pair(parameter_name_node, parameter_rvalue)
 
-    def save_parameters(cls, parameters_dictionary, compare_result, context):
+    def save_parameters(cls, parameters_dictionary, context):
+        '''Метод для сохранения значений параметров.'''
         context.parent['__parameters__'].set_parameters(parameters_dictionary)
-        context.parent['__parameters__'].set_condition(compare_result)
         return ''
 
 
-class TemplateEngine():
+class TemplateEngine:
     def __init__(self, directory_path='/',
                  parameters_set=set(),
                  env_set=set(),
                  datavars_module=Variables()):
         CalculateExtension._parameters_set = parameters_set
+        CalculateExtension._datavars = datavars_module
 
         self._datavars_module = datavars_module
         self._parameters_object = Parameters()
@@ -264,32 +312,30 @@ class TemplateEngine():
         self.environment.context_class = CalculateContext
 
     def change_directory(self, directory_path):
+        '''Метод для смены директории в загрузчике.'''
         self.environment.loader = FileSystemLoader(directory_path)
 
     def process_template(self, template_path, env=set()):
+        '''Метод для обработки файла шаблона, расположенного по указанному
+        пути.'''
         CalculateContext._env_set = env
         template = self.environment.get_template(template_path)
-        self._parameters_object = Parameters(parameters_dictionary={},
-                                             condition=True)
+        self._parameters_object = Parameters(parameters_dictionary={})
         self._template_text = template.render(
                                          __datavars__=self._datavars_module,
                                          __parameters__=self._parameters_object
                                              )
 
     def process_template_from_string(self, string, env=set()):
+        '''Метод для обработки текста шаблона.'''
         CalculateContext._env_set = env
         template = self.environment.from_string(string)
-        self._parameters_object = Parameters(parameters_dictionary={},
-                                             condition=True)
+        self._parameters_object = Parameters(parameters_dictionary={})
         self._template_text = template.render(
                                          __datavars__=self._datavars_module,
                                          __parameters__=self._parameters_object
                                              )
 
-    @property
-    def condition(self):
-        return self._parameters_object.condition
-
     @property
     def parameters(self):
         return self._parameters_object.parameters

calculate/utils/files.py

@@ -1,9 +1,12 @@
 # vim: fileencoding=utf-8
 #
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, STDOUT
 from io import TextIOWrapper
 from os import path
+from .tools import GenericFS, get_traceback_caller
+from glob import glob
 import os
+import sys
 
 
 class FilesError(Exception):
@@ -35,6 +38,9 @@ class KeyboardInputProcess():
 
 
 class Process():
+    STDOUT = STDOUT
+    PIPE = PIPE
+
     def __init__(self, command, *parameters, **kwargs):
         if 'stdin' not in kwargs:
             self._stdin = PipeProcess()
@@ -54,7 +60,7 @@ class Process():
         self._timeout = kwargs.get('timeout', None)
         self._cwd = kwargs.get('cwd', None)
 
-        self._command = _get_program_path(command)
+        self._command = get_program_path(command)
         if not self._command:
             raise FilesError("command not found '{}'".format(command))
         self._command = [self._command, *parameters]
@@ -221,7 +227,7 @@ class Process():
         return self.return_code() != 0
 
 
-class ProgramPathCache():
+class ProgramPathCache:
     def __init__(self):
         self._cache = {}
 
@@ -246,16 +252,195 @@ class ProgramPathCache():
         return False
 
 
-_get_program_path = ProgramPathCache()
+get_program_path = ProgramPathCache()
+
+
+def check_utils(*utils):
+    output = []
+    for util in utils:
+        util_path = get_program_path(util)
+        if not util_path:
+            raise FilesError("Command not found '{}'".
+                             format(os.path.basename(util)))
+        output.append(util)
+    if len(output) == 1:
+        return output[0]
+    else:
+        return output
 
 
 def join_paths(*paths):
     if len(paths) == 1:
         return next(iter(paths))
 
-    paths_to_join = filter(lambda path: path.strip() and path != "/",
-                           map(lambda path:
-                               path[1:] if path.startswith('/') else path,
-                               paths[1:]))
+    paths_to_join = []
+    for _path in paths[1:]:
+        if _path.startswith('/'):
+            _path = _path.strip()[1:]
+        else:
+            _path = _path.strip()
+        if _path and _path != "/":
+            paths_to_join.append(_path)
+
     output_path = path.join(paths[0], *paths_to_join)
     return output_path
+
+
+def read_file(file_path):
+    try:
+        if path.exists(file_path):
+            with open(file_path, 'r') as opened_file:
+                return opened_file.read()
+    except (OSError, IOError) as error:
+        mod, lineno = get_traceback_caller(*sys.exc_info())
+        sys.stderr.write("WARNING: file read error, {}({}:{})\n".
+                         format(str(error), mod, lineno))
+        sys.stderr.flush()
+    return ''
+
+
+def write_file(file_path):
+    directory_path = path.dirname(file_path)
+    if not path.exists(directory_path):
+        os.makedirs(directory_path)
+    return open(file_path, 'w')
+
+
+def read_file_lines(file_name, grab=False):
+    try:
+        if path.exists(file_name):
+            for file_line in open(file_name, 'r'):
+                if grab:
+                    file_line = file_line.strip()
+                    if not file_line.startswith('#'):
+                        yield file_line
+                else:
+                    yield file_line.rstrip('\n')
+    except (OSError, IOError):
+        pass
+    finally:
+        raise StopIteration
+
+
+def quite_unlink(file_path):
+    try:
+        if path.lexists(file_path):
+            os.unlink(file_path)
+    except OSError:
+        pass
+
+
+def list_directory(directory_path, full_path=False, only_dir=False):
+    if not path.exists(directory_path):
+        return []
+    try:
+        if full_path:
+            if only_dir:
+                return [node.path for node in os.scandir(directory_path)
+                        if os.path.isdir(node.path)]
+            else:
+                return [node.path for node in os.scandir(directory_path)]
+        else:
+            if only_dir:
+                return [node.name for node in os.scandir(directory_path)
+                        if os.path.isdir(node.path)]
+            else:
+                return os.listdir(directory_path)
+    except OSError:
+        return []
+
+
+def make_directory(directory_path, force=False):
+    try:
+        parent = os.path.split(path.normpath(directory_path))[0]
+        if not path.exists(parent):
+            make_directory(parent)
+        else:
+            if os.path.exists(directory_path):
+                if force and not os.path.isdir(directory_path):
+                    os.remove(directory_path)
+                else:
+                    return True
+        os.mkdir(directory_path)
+        return True
+    except (OSError, IOError):
+        return False
+
+
+class RealFS(GenericFS):
+    def __init__(self, prefix='/'):
+        self.prefix = prefix
+        if prefix == '/':
+            self.remove_prefix = lambda x: x
+        else:
+            self.remove_prefix = self._remove_prefix
+
+    def _remove_prefix(self, file_path):
+        prefix_length = len(self.prefix)
+        return file_path[:prefix_length]
+
+    def _get_path(self, file_path):
+        return join_paths(self.prefix, file_path)
+
+    def exists(self, file_path):
+        return os.path.lexists(self._get_path(file_path))
+
+    def read(self, file_path):
+        return read_file(self._get_path(file_path))
+
+    def glob(self, file_path):
+        for glob_path in glob(self._get_path(file_path)):
+            yield self.remove_prefix(glob_path)
+
+    def realpath(self, file_path):
+        return self.remove_prefix(path.realpath(file_path))
+
+    def write(self, file_path, data):
+        with write_file(file_path) as target_file:
+            target_file.write(data)
+
+    def listdir(self, file_path, full_path=False):
+        if full_path:
+            return [self.remove_prefix(listed_path)
+                    for listed_path in list_directory(file_path,
+                                                      full_path=full_path)]
+        else:
+            return list_directory(file_path, full_path=full_path)
+
+
+def get_run_commands(not_chroot=False, chroot=None, uid=None, with_pid=False):
+    def get_cmdline(process_number):
+        cmdline_file = '/proc/{}/cmdline'.format(process_number)
+        try:
+            if uid is not None:
+                fstat = os.stat('/proc/{}'.format(process_number))
+                if fstat.st_uid != uid:
+                    return ''
+            if path.exists(cmdline_file):
+                if not_chroot:
+                    root_link = '/proc/{}/root'.format(process_number)
+                    if os.readlink(root_link) != '/':
+                        return ''
+                if chroot is not None:
+                    root_link = '/proc/{}/root'.format(process_number)
+                    if os.readlink(root_link) != chroot:
+                        return ''
+                return read_file(cmdline_file).strip()
+        except Exception:
+            pass
+        return ''
+
+    if not os.access('/proc', os.R_OK):
+        return []
+
+    proc_directory = list_directory('/proc')
+    output = []
+    for file_name in proc_directory:
+        if file_name.isdigit():
+            cmdline = get_cmdline(file_name)
+            if cmdline:
+                if with_pid:
+                    output.append((file_name, cmdline))
+                else:
+                    output.append(cmdline)
+    return output

pytest.ini

@@ -2,7 +2,6 @@
 # pytest.ini
 [pytest]
 markers =
-    vars: marker for running tests for datavars
     base: marker for running tests for base format class.
     bind: marker for running tests for bind format.
     compiz: marker for running tests for compiz format.
@@ -20,5 +19,9 @@ markers =
     samba: marker for running tests for samba format.
     xml_xfce: marker for running tests for xml xfce format.
     xml_gconf: marker for running tests for xml gconf format.
+
     files: marker for running tests for calculate.utils.files module.
-    template_engine: marker for running tests for template_engine.
+    vars: marker for running tests for datavars
+    template_engine: marker for running tests for TemplateEngine.
+    directory_processor: marker for running tests for DirectoryProcessor.
+    template_action: marker for running tests for TemplateAction.

tests/templates/format/test_diff.py

@@ -16,6 +16,8 @@ class TestExecuteMethods:
         diff_patch = DiffFormat(patch_text)
         print('Path:', root_path)
         output = diff_patch.execute_format(root_path=root_path)
+        print('Output:')
+        print(output)
         if output:
             print('Changed files:')
             for changed_file in diff_patch._changed_files_list:

tests/templates/test_template_engine.py

@@ -1,5 +1,6 @@
 import pytest
-from calculate.templates.template_engine import TemplateEngine, Variables
+from calculate.templates.template_engine import TemplateEngine, Variables,\
+                                                ConditionFailed
 
 
 PARAMETERS_SET = {'name', 'path', 'append', 'chmod', 'chown',
@@ -11,7 +12,7 @@ PARAMETERS_SET = {'name', 'path', 'append', 'chmod', 'chown',
 
 
 @pytest.mark.template_engine
-class TestUtils():
+class TestTemplateEngine():
     def test_if_an_input_template_contains_calculate_tag_with_some_parameters__the_template_engine_object_will_collect_its_parameters(self):
         input_template = '''{% calculate name = 'filename', path = '/etc/path', force %}'''
         parameters = {'name': 'filename', 'path': '/etc/path', 'force': True}
@@ -48,7 +49,7 @@ class TestUtils():
         output_parameters = template_engine.parameters
         assert output_parameters == parameters
 
-    def test_if_an_input_template_contains_condition__the_template_engine_object_will_contain_the_value_of_its_condition(self):
+    def test_if_an_input_template_contains_condition_and_it_is_True__the_template_engine_object_will_be_initialized_without_any_exceptions(self):
         input_template = '''{% calculate vars.var_1 < vars.var_2 or (not (var_3 == 'required status') and vars.var_4), env = 'vars' %}'''
         datavars_module = Variables({'vars':
                                      Variables({'var_1': 12,
@@ -57,14 +58,31 @@ class TestUtils():
                                                 'var_4': True})})
         template_engine = TemplateEngine(parameters_set=PARAMETERS_SET,
                                          datavars_module=datavars_module)
-        template_engine.process_template_from_string(input_template)
-        condition = template_engine.condition
-        assert condition is True
+        try:
+            template_engine.process_template_from_string(input_template)
+        except ConditionFailed:
+            pytest.fail('Unexpected ConditionFailed exception.')
 
-    def test_if_an_input_template_contains_several_calculate_tags__the_template_engine_will_parse_them_all_and_will_contain_all_parameters_and_result_of_all_conditions(self):
+    def test_if_an_input_template_contains_several_conditions_and_it_is_False__the_template_engine_raises_ConditionFailed_exception(self):
         input_template = '''{% calculate name = vars.var_1, var_4 < var_5 %}
                             {% calculate path = var_3, var_6 == 'value' %}
                             {% calculate env = 'other_vars'%}'''
+        datavars_module = Variables({'vars':
+                                     Variables({'var_1': 'filename',
+                                                'var_2': '/etc/path'}),
+                                     'other_vars':
+                                     Variables({'var_3': '/etc/other_path',
+                                                'var_4': 12, 'var_5': 1.2,
+                                                'var_6': 'value'})})
+        template_engine = TemplateEngine(parameters_set=PARAMETERS_SET,
+                                         datavars_module=datavars_module)
+        with pytest.raises(ConditionFailed):
+            template_engine.process_template_from_string(input_template)
+
+    def test_if_an_input_template_contains_several_calculate_tags__the_template_engine_will_parse_them_all_and_will_contain_all_parameters_and_result_of_all_conditions(self):
+        input_template = '''{% calculate name = vars.var_1, var_4 > var_5 %}
+                            {% calculate path = var_3, var_6 == 'value' %}
+                            {% calculate env = 'other_vars'%}'''
         parameters = {'name': 'filename', 'path': '/etc/other_path',
                       'env': 'other_vars'}
         datavars_module = Variables({'vars':
@@ -77,9 +95,7 @@ class TestUtils():
         template_engine = TemplateEngine(parameters_set=PARAMETERS_SET,
                                          datavars_module=datavars_module)
         template_engine.process_template_from_string(input_template)
-        condition = template_engine.condition
-        output_parameters = template_engine.parameters
-        assert condition is False and output_parameters == parameters
+        assert template_engine.parameters == parameters
 
     def test_if_an_input_template_contains_variables_in_its_text__the_rendered_text_will_contain_values_of_this_variables(self):
         input_template = '''{% calculate name = 'filename', force -%}