diff --git a/calculate/templates/format/base_format.py b/calculate/templates/format/base_format.py index 9648f6b..473aab1 100644 --- a/calculate/templates/format/base_format.py +++ b/calculate/templates/format/base_format.py @@ -10,6 +10,10 @@ except ImportError: from xml.etree.ElementTree import fromstring +class FormatError(Exception): + pass + + class BaseFormat(): def __init__(self, processing_methods): self._processing_methods = processing_methods diff --git a/calculate/templates/format/diff_format.py b/calculate/templates/format/diff_format.py index e7fb107..f04627e 100644 --- a/calculate/templates/format/diff_format.py +++ b/calculate/templates/format/diff_format.py @@ -1,6 +1,7 @@ # vim: fileencoding=utf-8 # from calculate.utils.files import Process +from calculate.templates.format.base_format import FormatError from os import path @@ -14,21 +15,16 @@ class DiffFormat(): self._changed_files_list = [] def execute_format(self, root_path): + print if path.exists(root_path): self._root_path = root_path else: - # Какая-то обработка ошибки. - error_message = 'Root path does not exist.' - print(error_message) - return False + raise FormatError('Root path does not exist.') if self._patch_text: return self._patch_document() else: - # Какая-то обработка ошибки. - error_message = 'Empty patch file.' - print(error_message) - return False + raise FormatError('Empty patch file.') def _patch_document(self): for level in range(0, 4): @@ -44,19 +40,19 @@ class DiffFormat(): if patch_dry_run.success(): return '' else: - # Какая-то обработка ошибки. - error_message = 'Correction failed.' - print(error_message) - return False + raise FormatError('Correction failed.') self._last_level = level patch_run = Process('patch', '-p{}'.format(level), cwd=self._root_path) patch_run.write(self._patch_text) if patch_run.success(): + print('patch run is successful...') for line in patch_run: if line.startswith('patching file'): self._changed_files_list.append(line[13:].strip()) return patch_run.read() else: + print('patch run is no successful...') + print(patch_run.read_error()) return '' diff --git a/calculate/templates/template_engine.py b/calculate/templates/template_engine.py index 8e339a6..b96370f 100644 --- a/calculate/templates/template_engine.py +++ b/calculate/templates/template_engine.py @@ -44,6 +44,8 @@ class Variables(MutableMapping): def resolve_or_missing(context, key, missing=missing, env={}): + '''Переопределение функции из для поиска значений переменных из jinja2. + Ищет переменные в datavars.''' datavars = context.parent['__datavars__'] if key in context.vars: return context.vars[key] @@ -58,6 +60,7 @@ def resolve_or_missing(context, key, missing=missing, env={}): class CalculateContext(Context): + '''Класс контекста позволяющий искать значения datavars и сохранять их.''' _env_set = set() def resolve(self, key): @@ -80,20 +83,20 @@ class CalculateContext(Context): env=self._env_set) +class ConditionFailed(TemplateSyntaxError): + pass + + class Parameters(MutableMapping): - '''Класс для хранения параметров и условий, взятых из шаблона, и передачи + '''Класс для хранения параметров, взятых из шаблона, и передачи их шаблонизатору.''' - def __init__(self, parameters_dictionary={}, condition=True): + def __init__(self, parameters_dictionary={}): self.__parameters = parameters_dictionary - self.__condition = condition def set_parameters(self, *args, **kwargs): parameters = dict(*args, **kwargs) self.__parameters.update(parameters) - def set_condition(self, condition): - self.__condition = self.__condition and condition - def __getitem__(self, name): return self.__parameters[name] @@ -110,12 +113,7 @@ class Parameters(MutableMapping): return len(self.__parameters) def __repr__(self): - return ''.format(self.__parameters, - self.__condition) - - @property - def condition(self): - return self.__condition + return ''.format(self.__parameters) @property def parameters(self): @@ -123,7 +121,9 @@ class Parameters(MutableMapping): class CalculateExtension(Extension): + '''Класс расширения для jinja2, поддерживающий теги calculate-шаблонов.''' _parameters_set = set() + _datavars = Variables() def __init__(self, environment): self.tags = {'calculate', 'save', 'set_var'} @@ -134,6 +134,8 @@ class CalculateExtension(Extension): self.parse_methods = {'calculate': self.parse_calculate, 'save': self.parse_save} + self.environment = environment + def parse(self, parser): self.parser = parser self.stream = parser.stream @@ -141,6 +143,8 @@ class CalculateExtension(Extension): return [self.parse_methods[tag_token]()] def parse_save(self): + '''Метод для разбора тега save, сохраняющего значение указанной + переменной datavars.''' lineno = next(self.stream).lineno target_file = nodes.Const('', lineno=lineno) @@ -175,9 +179,10 @@ class CalculateExtension(Extension): lineno=lineno) def parse_calculate(self): + '''Метод для разбора тега calculate, содержащего значения параметров и + условия выполнения шаблона.''' pairs_list = [] expect_comma_flag = False - conditions = nodes.Const(True) lineno = next(self.stream).lineno @@ -185,17 +190,22 @@ class CalculateExtension(Extension): if expect_comma_flag: self.stream.expect('comma') - if self.stream.current.type == 'name': - if (self.stream.current.value in self._parameters_set and - self.stream.look().type != 'dot'): - pairs_list.append(self.get_parameter_node()) - else: - conditions = nodes.And( - self.parser.parse_expression( - with_condexpr=True - ), - conditions - ) + if (self.stream.current.type == 'name' + and self.stream.current.value in self._parameters_set + and self.stream.look().type != 'dot' + and self.stream.look().type not in + self.CONDITION_TOKENS_TYPES): + # разбираем параметр. + pairs_list.append(self.get_parameter_node()) + elif (self.stream.current.type == 'name' + or self.stream.current.type == 'lparen'): + # разбираем условие. Если условие False -- кидаем исключение. + condition_result = self.get_condition_result() + if not condition_result: + raise ConditionFailed( + 'Condition is failed', + lineno=self.stream.current.lineno + ) else: raise TemplateSyntaxError('Name is expected in calculate tag.', lineno=self.stream.current.lineno) @@ -203,13 +213,47 @@ class CalculateExtension(Extension): dictionary_node = nodes.Dict(pairs_list) save_node = self.call_method('save_parameters', [dictionary_node, - conditions, nodes.ContextReference()], lineno=lineno) return nodes.Output([save_node], lineno=lineno) + def get_condition_result(self): + '''Метод для разбора условий из тега calculate.''' + condition_list = [] + # собираем исходный код условия из токенов. + # вероятно, следует придумать лучший способ. + while (self.stream.current.type != 'block_end' and + self.stream.current.type != 'comma'): + if self.stream.current.type == 'string': + condition_list.append("'{}'".format( + self.stream.current.value + )) + elif self.stream.current.type == 'dot': + self.stream.skip(1) + if self.stream.current.type == 'name': + next_name = '.' + self.stream.current.value + else: + raise TemplateSyntaxError( + 'Variable name is not correct.', + lineno=self.stream.current.lineno + ) + condition_list[-1] = condition_list[-1] + next_name + else: + condition_list.append( + str(self.stream.current.value) + ) + self.stream.skip(1) + + condition = ' '.join(condition_list) + + # компилируем исходный код условия и получаем результат его вычисления. + cond_expr = self.environment.compile_expression(condition) + condition_result = cond_expr(__datavars__=self._datavars) + return condition_result + def save_variable(self, variable_name, right_value, target_file, context): - '''Временный метод для сохранения значений переменных.''' + '''Метод для сохранения значений переменных указанных в теге save.''' + # временная реализация. datavars = context.parent['__datavars__'] module_name = variable_name[0] namespaces = variable_name[1:-1] @@ -228,6 +272,7 @@ class CalculateExtension(Extension): return '' def get_parameter_node(self): + '''Метод для разбра параметров, содержащихся в теге calculate.''' lineno = self.stream.current.lineno parameter_name = self.stream.expect('name').value parameter_name_node = nodes.Const(parameter_name, lineno=lineno) @@ -235,6 +280,8 @@ class CalculateExtension(Extension): parameter_value = self.stream.current.value parameter_rvalue = self.parser.parse_expression(with_condexpr=True) if parameter_name == 'env': + # если параметр env -- обновляем множенство значений env + # контекста вo время парсинга. env_names = parameter_value.split(',') for name in env_names: CalculateContext._env_set.add(name.strip()) @@ -242,18 +289,19 @@ class CalculateExtension(Extension): parameter_rvalue = nodes.Const(True, lineno=lineno) return nodes.Pair(parameter_name_node, parameter_rvalue) - def save_parameters(cls, parameters_dictionary, compare_result, context): + def save_parameters(cls, parameters_dictionary, context): + '''Метод для сохранения значений параметров.''' context.parent['__parameters__'].set_parameters(parameters_dictionary) - context.parent['__parameters__'].set_condition(compare_result) return '' -class TemplateEngine(): +class TemplateEngine: def __init__(self, directory_path='/', parameters_set=set(), env_set=set(), datavars_module=Variables()): CalculateExtension._parameters_set = parameters_set + CalculateExtension._datavars = datavars_module self._datavars_module = datavars_module self._parameters_object = Parameters() @@ -264,32 +312,30 @@ class TemplateEngine(): self.environment.context_class = CalculateContext def change_directory(self, directory_path): + '''Метод для смены директории в загрузчике.''' self.environment.loader = FileSystemLoader(directory_path) def process_template(self, template_path, env=set()): + '''Метод для обработки файла шаблона, расположенного по указанному + пути.''' CalculateContext._env_set = env template = self.environment.get_template(template_path) - self._parameters_object = Parameters(parameters_dictionary={}, - condition=True) + self._parameters_object = Parameters(parameters_dictionary={}) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object ) def process_template_from_string(self, string, env=set()): + '''Метод для обработки текста шаблона.''' CalculateContext._env_set = env template = self.environment.from_string(string) - self._parameters_object = Parameters(parameters_dictionary={}, - condition=True) + self._parameters_object = Parameters(parameters_dictionary={}) self._template_text = template.render( __datavars__=self._datavars_module, __parameters__=self._parameters_object ) - @property - def condition(self): - return self._parameters_object.condition - @property def parameters(self): return self._parameters_object.parameters diff --git a/calculate/utils/files.py b/calculate/utils/files.py index 7cefa06..92f8720 100644 --- a/calculate/utils/files.py +++ b/calculate/utils/files.py @@ -1,9 +1,12 @@ # vim: fileencoding=utf-8 # -from subprocess import Popen, PIPE +from subprocess import Popen, PIPE, STDOUT from io import TextIOWrapper from os import path +from .tools import GenericFS, get_traceback_caller +from glob import glob import os +import sys class FilesError(Exception): @@ -35,6 +38,9 @@ class KeyboardInputProcess(): class Process(): + STDOUT = STDOUT + PIPE = PIPE + def __init__(self, command, *parameters, **kwargs): if 'stdin' not in kwargs: self._stdin = PipeProcess() @@ -54,7 +60,7 @@ class Process(): self._timeout = kwargs.get('timeout', None) self._cwd = kwargs.get('cwd', None) - self._command = _get_program_path(command) + self._command = get_program_path(command) if not self._command: raise FilesError("command not found '{}'".format(command)) self._command = [self._command, *parameters] @@ -221,7 +227,7 @@ class Process(): return self.return_code() != 0 -class ProgramPathCache(): +class ProgramPathCache: def __init__(self): self._cache = {} @@ -246,16 +252,195 @@ class ProgramPathCache(): return False -_get_program_path = ProgramPathCache() +get_program_path = ProgramPathCache() + + +def check_utils(*utils): + output = [] + for util in utils: + util_path = get_program_path(util) + if not util_path: + raise FilesError("Command not found '{}'". + format(os.path.basename(util))) + output.append(util) + if len(output) == 1: + return output[0] + else: + return output def join_paths(*paths): if len(paths) == 1: return next(iter(paths)) - paths_to_join = filter(lambda path: path.strip() and path != "/", - map(lambda path: - path[1:] if path.startswith('/') else path, - paths[1:])) + paths_to_join = [] + for _path in paths[1:]: + if _path.startswith('/'): + _path = _path.strip()[1:] + else: + _path = _path.strip() + if _path and _path != "/": + paths_to_join.append(_path) + output_path = path.join(paths[0], *paths_to_join) return output_path + + +def read_file(file_path): + try: + if path.exists(file_path): + with open(file_path, 'r') as opened_file: + return opened_file.read() + except (OSError, IOError) as error: + mod, lineno = get_traceback_caller(*sys.exc_info()) + sys.stderr.write("WARNING: file read error, {}({}:{})\n". + format(str(error), mod, lineno)) + sys.stderr.flush() + return '' + + +def write_file(file_path): + directory_path = path.dirname(file_path) + if not path.exists(directory_path): + os.makedirs(directory_path) + return open(file_path, 'w') + + +def read_file_lines(file_name, grab=False): + try: + if path.exists(file_name): + for file_line in open(file_name, 'r'): + if grab: + file_line = file_line.strip() + if not file_line.startswith('#'): + yield file_line + else: + yield file_line.rstrip('\n') + except (OSError, IOError): + pass + finally: + raise StopIteration + + +def quite_unlink(file_path): + try: + if path.lexists(file_path): + os.unlink(file_path) + except OSError: + pass + + +def list_directory(directory_path, full_path=False, only_dir=False): + if not path.exists(directory_path): + return [] + try: + if full_path: + if only_dir: + return [node.path for node in os.scandir(directory_path) + if os.path.isdir(node.path)] + else: + return [node.path for node in os.scandir(directory_path)] + else: + if only_dir: + return [node.name for node in os.scandir(directory_path) + if os.path.isdir(node.path)] + else: + return os.listdir(directory_path) + except OSError: + return [] + + +def make_directory(directory_path, force=False): + try: + parent = os.path.split(path.normpath(directory_path))[0] + if not path.exists(parent): + make_directory(parent) + else: + if os.path.exists(directory_path): + if force and not os.path.isdir(directory_path): + os.remove(directory_path) + else: + return True + os.mkdir(directory_path) + return True + except (OSError, IOError): + return False + + +class RealFS(GenericFS): + def __init__(self, prefix='/'): + self.prefix = prefix + if prefix == '/': + self.remove_prefix = lambda x: x + else: + self.remove_prefix = self._remove_prefix + + def _remove_prefix(self, file_path): + prefix_length = len(self.prefix) + return file_path[:prefix_length] + + def _get_path(self, file_path): + return join_paths(self.prefix, file_path) + + def exists(self, file_path): + return os.path.lexists(self._get_path(file_path)) + + def read(self, file_path): + return read_file(self._get_path(file_path)) + + def glob(self, file_path): + for glob_path in glob(self._get_path(file_path)): + yield self.remove_prefix(glob_path) + + def realpath(self, file_path): + return self.remove_prefix(path.realpath(file_path)) + + def write(self, file_path, data): + with write_file(file_path) as target_file: + target_file.write(data) + + def listdir(self, file_path, full_path=False): + if full_path: + return [self.remove_prefix(listed_path) + for listed_path in list_directory(file_path, + full_path=full_path)] + else: + return list_directory(file_path, full_path=full_path) + + +def get_run_commands(not_chroot=False, chroot=None, uid=None, with_pid=False): + def get_cmdline(process_number): + cmdline_file = '/proc/{}/cmdline'.format(process_number) + try: + if uid is not None: + fstat = os.stat('/proc/{}'.format(process_number)) + if fstat.st_uid != uid: + return '' + if path.exists(cmdline_file): + if not_chroot: + root_link = '/proc/{}/root'.format(process_number) + if os.readlink(root_link) != '/': + return '' + if chroot is not None: + root_link = '/proc/{}/root'.format(process_number) + if os.readlink(root_link) != chroot: + return '' + return read_file(cmdline_file).strip() + except Exception: + pass + return '' + + if not os.access('/proc', os.R_OK): + return [] + + proc_directory = list_directory('/proc') + output = [] + for file_name in proc_directory: + if file_name.isdigit(): + cmdline = get_cmdline(file_name) + if cmdline: + if with_pid: + output.append((file_name, cmdline)) + else: + output.append(cmdline) + return output diff --git a/calculate/vars/os/__init__.py b/calculate/vars/os/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pytest.ini b/pytest.ini index 8dd47a3..1de1b1a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,6 @@ # pytest.ini [pytest] markers = - vars: marker for running tests for datavars base: marker for running tests for base format class. bind: marker for running tests for bind format. compiz: marker for running tests for compiz format. @@ -20,5 +19,9 @@ markers = samba: marker for running tests for samba format. xml_xfce: marker for running tests for xml xfce format. xml_gconf: marker for running tests for xml gconf format. + files: marker for running tests for calculate.utils.files module. - template_engine: marker for running tests for template_engine. + vars: marker for running tests for datavars + template_engine: marker for running tests for TemplateEngine. + directory_processor: marker for running tests for DirectoryProcessor. + template_action: marker for running tests for TemplateAction. diff --git a/tests/templates/format/test_diff.py b/tests/templates/format/test_diff.py index 87bc4f9..5c9a931 100644 --- a/tests/templates/format/test_diff.py +++ b/tests/templates/format/test_diff.py @@ -16,6 +16,8 @@ class TestExecuteMethods: diff_patch = DiffFormat(patch_text) print('Path:', root_path) output = diff_patch.execute_format(root_path=root_path) + print('Output:') + print(output) if output: print('Changed files:') for changed_file in diff_patch._changed_files_list: diff --git a/tests/templates/test_template_engine.py b/tests/templates/test_template_engine.py index 945e4a2..566b4e3 100644 --- a/tests/templates/test_template_engine.py +++ b/tests/templates/test_template_engine.py @@ -1,5 +1,6 @@ import pytest -from calculate.templates.template_engine import TemplateEngine, Variables +from calculate.templates.template_engine import TemplateEngine, Variables,\ + ConditionFailed PARAMETERS_SET = {'name', 'path', 'append', 'chmod', 'chown', @@ -11,7 +12,7 @@ PARAMETERS_SET = {'name', 'path', 'append', 'chmod', 'chown', @pytest.mark.template_engine -class TestUtils(): +class TestTemplateEngine(): def test_if_an_input_template_contains_calculate_tag_with_some_parameters__the_template_engine_object_will_collect_its_parameters(self): input_template = '''{% calculate name = 'filename', path = '/etc/path', force %}''' parameters = {'name': 'filename', 'path': '/etc/path', 'force': True} @@ -48,7 +49,7 @@ class TestUtils(): output_parameters = template_engine.parameters assert output_parameters == parameters - def test_if_an_input_template_contains_condition__the_template_engine_object_will_contain_the_value_of_its_condition(self): + def test_if_an_input_template_contains_condition_and_it_is_True__the_template_engine_object_will_be_initialized_without_any_exceptions(self): input_template = '''{% calculate vars.var_1 < vars.var_2 or (not (var_3 == 'required status') and vars.var_4), env = 'vars' %}''' datavars_module = Variables({'vars': Variables({'var_1': 12, @@ -57,14 +58,31 @@ class TestUtils(): 'var_4': True})}) template_engine = TemplateEngine(parameters_set=PARAMETERS_SET, datavars_module=datavars_module) - template_engine.process_template_from_string(input_template) - condition = template_engine.condition - assert condition is True + try: + template_engine.process_template_from_string(input_template) + except ConditionFailed: + pytest.fail('Unexpected ConditionFailed exception.') - def test_if_an_input_template_contains_several_calculate_tags__the_template_engine_will_parse_them_all_and_will_contain_all_parameters_and_result_of_all_conditions(self): + def test_if_an_input_template_contains_several_conditions_and_it_is_False__the_template_engine_raises_ConditionFailed_exception(self): input_template = '''{% calculate name = vars.var_1, var_4 < var_5 %} {% calculate path = var_3, var_6 == 'value' %} {% calculate env = 'other_vars'%}''' + datavars_module = Variables({'vars': + Variables({'var_1': 'filename', + 'var_2': '/etc/path'}), + 'other_vars': + Variables({'var_3': '/etc/other_path', + 'var_4': 12, 'var_5': 1.2, + 'var_6': 'value'})}) + template_engine = TemplateEngine(parameters_set=PARAMETERS_SET, + datavars_module=datavars_module) + with pytest.raises(ConditionFailed): + template_engine.process_template_from_string(input_template) + + def test_if_an_input_template_contains_several_calculate_tags__the_template_engine_will_parse_them_all_and_will_contain_all_parameters_and_result_of_all_conditions(self): + input_template = '''{% calculate name = vars.var_1, var_4 > var_5 %} + {% calculate path = var_3, var_6 == 'value' %} + {% calculate env = 'other_vars'%}''' parameters = {'name': 'filename', 'path': '/etc/other_path', 'env': 'other_vars'} datavars_module = Variables({'vars': @@ -77,9 +95,7 @@ class TestUtils(): template_engine = TemplateEngine(parameters_set=PARAMETERS_SET, datavars_module=datavars_module) template_engine.process_template_from_string(input_template) - condition = template_engine.condition - output_parameters = template_engine.parameters - assert condition is False and output_parameters == parameters + assert template_engine.parameters == parameters def test_if_an_input_template_contains_variables_in_its_text__the_rendered_text_will_contain_values_of_this_variables(self): input_template = '''{% calculate name = 'filename', force -%}