Browse Source

Added formats

packages
commit
fce2667231
  1. 5
      Makefile
  2. 0
      README.txt
  3. 0
      calculate/__init__.py
  4. 0
      calculate/templates/__init__.py
  5. 0
      calculate/templates/format/__init__.py
  6. 189
      calculate/templates/format/base_format.py
  7. 293
      calculate/templates/format/bind_format.py
  8. 150
      calculate/templates/format/compiz_format.py
  9. 62
      calculate/templates/format/diff_format.py
  10. 229
      calculate/templates/format/dovecot_format.py
  11. 26
      calculate/templates/format/json_format.py
  12. 155
      calculate/templates/format/kde_format.py
  13. 100
      calculate/templates/format/kernel_format.py
  14. 522
      calculate/templates/format/ldap_format.py
  15. 99
      calculate/templates/format/openrc_format.py
  16. 177
      calculate/templates/format/patch_format.py
  17. 98
      calculate/templates/format/postfix_format.py
  18. 99
      calculate/templates/format/procmail_format.py
  19. 411
      calculate/templates/format/proftpd_format.py
  20. 163
      calculate/templates/format/samba_format.py
  21. 27
      calculate/templates/format/templates/bind
  22. 17
      calculate/templates/format/templates/compiz
  23. 28
      calculate/templates/format/templates/dovecot
  24. 19
      calculate/templates/format/templates/kde
  25. 6
      calculate/templates/format/templates/kernel
  26. 59
      calculate/templates/format/templates/ldap
  27. 7
      calculate/templates/format/templates/openrc
  28. 8
      calculate/templates/format/templates/postfix
  29. 7
      calculate/templates/format/templates/procmail
  30. 38
      calculate/templates/format/templates/proftpd
  31. 16
      calculate/templates/format/templates/samba
  32. 14
      calculate/templates/format/templates/squid
  33. 223
      calculate/templates/format/xml_gconf_format.py
  34. 164
      calculate/templates/format/xml_xfce_format.py
  35. 0
      calculate/utils/__init__.py
  36. 261
      calculate/utils/files.py
  37. BIN
      calculate/vars/.datavars.py.swp
  38. 0
      calculate/vars/__init__.py
  39. 51
      calculate/vars/datavars.py
  40. 0
      calculate/vars/os/__init__.py
  41. 4
      calculate/vars/os/gentoo/__init__.py
  42. 164
      conftest.py
  43. 23
      pytest.ini
  44. 8
      setup.py
  45. 0
      tests/format/__init__.py
  46. 83
      tests/format/test_base.py
  47. 298
      tests/format/test_bind.py
  48. 218
      tests/format/test_compiz.py
  49. 97
      tests/format/test_diff.py
  50. 255
      tests/format/test_dovecot.py
  51. 44
      tests/format/test_json.py
  52. 222
      tests/format/test_kde.py
  53. 112
      tests/format/test_kernel.py
  54. 364
      tests/format/test_ldap.py
  55. 146
      tests/format/test_openrc.py
  56. 80
      tests/format/test_patch.py
  57. 124
      tests/format/test_postfix.py
  58. 130
      tests/format/test_procmail.py
  59. 237
      tests/format/test_proftpd.py
  60. 229
      tests/format/test_samba.py
  61. 89
      tests/format/test_xml_gconf.py
  62. 52
      tests/format/test_xml_xfce.py
  63. 10
      tests/format/testfiles/a/dir/file1.txt
  64. 10
      tests/format/testfiles/a/dir/file2.txt
  65. 10
      tests/format/testfiles/a1/dir/file1.txt
  66. 17
      tests/format/testfiles/b/dir/file1.txt
  67. 16
      tests/format/testfiles/b/dir/file2.txt
  68. 11
      tests/format/testfiles/b1/dir/file1.txt
  69. 16
      tests/format/testfiles/b1/dir/file2.txt
  70. 108
      tests/format/testfiles/bind_original.conf
  71. 131
      tests/format/testfiles/bind_result.conf
  72. 58
      tests/format/testfiles/bind_template.conf
  73. 32
      tests/format/testfiles/compiz_original
  74. 34
      tests/format/testfiles/compiz_result
  75. 39
      tests/format/testfiles/compiz_template
  76. 34
      tests/format/testfiles/diff_1.patch
  77. 41
      tests/format/testfiles/diff_2.patch
  78. 38
      tests/format/testfiles/dovecot_original.conf
  79. 47
      tests/format/testfiles/dovecot_result.conf
  80. 39
      tests/format/testfiles/dovecot_template.conf
  81. 21
      tests/format/testfiles/json_original.json
  82. 25
      tests/format/testfiles/json_result.json
  83. 14
      tests/format/testfiles/json_template.json
  84. 63
      tests/format/testfiles/kde_original
  85. 75
      tests/format/testfiles/kde_result
  86. 40
      tests/format/testfiles/kde_template
  87. 39
      tests/format/testfiles/kernel_original
  88. 34
      tests/format/testfiles/kernel_result
  89. 10
      tests/format/testfiles/kernel_template
  90. 9
      tests/format/testfiles/ldap_logic_lines_test.txt
  91. 71
      tests/format/testfiles/ldap_original.conf
  92. 67
      tests/format/testfiles/ldap_result.conf
  93. 71
      tests/format/testfiles/ldap_template.conf
  94. 5
      tests/format/testfiles/logic_lines_test.txt
  95. 41
      tests/format/testfiles/logic_lines_test_input.txt
  96. 39
      tests/format/testfiles/logic_lines_test_output.txt
  97. 37
      tests/format/testfiles/openrc_original
  98. 37
      tests/format/testfiles/openrc_result
  99. 15
      tests/format/testfiles/openrc_template
  100. 34
      tests/format/testfiles/postfix_original

5
Makefile

@ -0,0 +1,5 @@
install:
sudo python ./setup.py install --single-version-externally-managed --root=/
test: install
pytest

0
README.txt

0
calculate/__init__.py

0
calculate/templates/__init__.py

0
calculate/templates/format/__init__.py

189
calculate/templates/format/base_format.py

@ -0,0 +1,189 @@
# vim: fileencoding=utf-8
#
from collections import OrderedDict
from jinja2 import Environment, PackageLoader
from pprint import pprint
try:
from lxml.etree.ElementTree import fromstring
except ImportError:
from xml.etree.ElementTree import fromstring
class BaseFormat():
def __init__(self, processing_methods):
self._processing_methods = processing_methods
self._document_dictionary = OrderedDict()
self._item_to_add = OrderedDict()
self._format = 'none'
self.TEMPLATES_DIRECTORY = 'templates'
self._fatal_error_flag = False
self._ready_to_update = False
self._match = False
self._need_finish = False
self._comments_processing = False
self._join_before = False
self._join_before_in_areas = False
# для отладки.
self._line_timer = 0
def _lines_to_dictionary(self, document_lines):
# print('Lines processing...')
for line in document_lines:
# print(self._line_timer, '\t', line)
for processing_method in self._processing_methods:
processing_method(line)
if self._fatal_error_flag:
# Действия если файл невозможно разобрать.
print('Can not parse file.')
self._document_dictionary = OrderedDict()
return
if self._is_match():
if self._is_ready_to_update():
self._document_dictionary.update(self._item_to_add)
break
else:
# Действия если не удалось разобрать строку.
print('Line', self._line_timer,
'is not correct. Can not parse file.')
self._document_dictionary = OrderedDict()
return
self._line_timer += 1
if self._need_finish:
self._finish_method()
def _parse_xml_to_dictionary(self, xml_document_text):
root = fromstring(xml_document_text)
self._document_dictionary = self._processing_methods[root.tag](root)
def print_dictionary(self):
pprint(self._document_dictionary)
def join_template(self, template):
self._join(self._document_dictionary,
template._document_dictionary,
self._join_before)
def _get_list_of_logic_lines(self, text):
list_of_lines = []
lines_to_join = []
for line in text.splitlines():
line = line.strip()
if line == '':
continue
if not line.endswith("\\"):
lines_to_join.append(line)
joined_line = "".join(lines_to_join)
list_of_lines.append(joined_line)
lines_to_join = []
else:
lines_to_join.append(line[:-1])
return list_of_lines
def _join(self, original, template, join_before):
if template == OrderedDict():
return
if join_before:
forwarded_items = OrderedDict()
for key_value in template:
if key_value[0] == '!':
# Удаление соответствующего элемента из original.
if isinstance(key_value, tuple):
item_to_delete = ('',) + key_value[1:]
elif isinstance(key_value, str):
item_to_delete = key_value[1:]
if item_to_delete in original.keys():
original.pop(item_to_delete)
elif key_value[0] == '-':
# Замена соответствующего элемента из original.
if isinstance(key_value, tuple):
item_to_replace = ('',) + key_value[1:]
elif isinstance(key_value, str):
item_to_replace = key_value[1:]
if item_to_replace not in original.keys():
continue
if isinstance(template[key_value], dict) and\
template[key_value] == OrderedDict():
original.pop(item_to_replace)
continue
if self._comments_processing:
if '#' in original[item_to_replace]:
replaced = OrderedDict({'#':
original[item_to_replace]['#']}
)
replaced.update(template[key_value])
else:
replaced = template[key_value]
original[item_to_replace] = replaced
else:
original[item_to_replace] = template[key_value]
elif key_value not in original.keys():
if isinstance(template[key_value], dict):
dictionary_to_add = OrderedDict()
self._join(dictionary_to_add,
template[key_value],
self._join_before_in_areas)
if dictionary_to_add != OrderedDict():
if not join_before:
original[key_value] = dictionary_to_add
else:
forwarded_items[key_value] = dictionary_to_add
else:
if not join_before:
original[key_value] = template[key_value]
else:
forwarded_items[key_value] = template[key_value]
else:
if isinstance(original[key_value], dict) and \
isinstance(template[key_value], dict):
self._join(original[key_value],
template[key_value],
self._join_before_in_areas)
else:
if self._comments_processing:
original[key_value][-1] = template[key_value][-1]
else:
original[key_value] = template[key_value]
if join_before:
for key_value in reversed(forwarded_items.keys()):
original[key_value] = forwarded_items[key_value]
original.move_to_end(key_value, last=False)
def get_document_text(self):
file_loader = PackageLoader('calculate.templates.format',
self.TEMPLATES_DIRECTORY)
formats_environment = Environment(loader=file_loader,
trim_blocks=True,
lstrip_blocks=True)
formats_environment.globals.update(zip=zip)
formats_environment.add_extension('jinja2.ext.do')
template = formats_environment.get_template(self._format)
document_text = template.render(
document_dictionary=self._document_dictionary
)
return document_text
def _finish_method(self):
pass
def _is_ready_to_update(self):
is_ready, self._match = self._ready_to_update, False
return is_ready
def _is_match(self):
is_match, self._match = self._match, False
return is_match

293
calculate/templates/format/bind_format.py

@ -0,0 +1,293 @@
# vim: fileencoding=utf-8
#
# ToDo: добавить проверку того, полностью ли парсился документ. Если отпарсился
# не весь файл -- выдаем ошибку.
#
from .base_format import BaseFormat
from collections import OrderedDict
from pyparsing import originalTextFor, OneOrMore, Word, alphanums, Literal,\
ZeroOrMore, Forward, Optional, Group, restOfLine,\
cppStyleComment, Keyword, printables, nums, SkipTo
class BINDFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = []
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._comments_processing = True
self._format = 'bind'
self._last_comments_list = []
self._initialize_parser()
if document_text == '':
self._document_dictionary = OrderedDict()
else:
self._parse_text(document_text)
def _initialize_parser(self):
left_brace = Literal('{')
right_brace = Literal('}')
semicolon = Literal(';')
action_symbols = (Literal('!') |
Literal('-'))
plain_allow = Keyword('allow')
drop_allow = Keyword('!allow')
replace_allow = Keyword('-allow')
allow = (plain_allow | drop_allow | replace_allow)
keys = Keyword('keys')
inet = Keyword('inet')
statement = originalTextFor(
Word(alphanums+'_-',
excludeChars='{};')
)
statement_name = originalTextFor(
Word(printables,
excludeChars='{};')
)
statement_class = originalTextFor(
Word(printables,
excludeChars='{};')
)
parameter_value = originalTextFor(
Word(printables,
excludeChars='{};')
)('parameter')
ip_value = originalTextFor(
Word(nums+':./',
excludeChars=';{}')
)
# Будущий парсер блока.
block = Forward()
# Для парсинга директивы inet_spec.
allow_item = (ip_value | statement_name) + semicolon.suppress()
key_item = statement_name + semicolon.suppress()
allow_group = Group(Group(Optional(action_symbols,
default='')('action')
+ allow)
+ Group(left_brace.suppress()
+ ZeroOrMore(allow_item)
+ right_brace.suppress()))
keys_group = Group(Group(Optional(action_symbols, default='')('action')
+ keys)
+ Group(left_brace.suppress()
+ ZeroOrMore(key_item)
+ right_brace.suppress()))
inet_spec = (Group(Optional(action_symbols, default='')('action')
+ inet + originalTextFor(SkipTo(allow,
include=False)
)('parameter')
)('name')
+ Group(allow_group
+ Optional(keys_group)
+ semicolon.suppress())('content')
).setParseAction(
self._add_inet_specline
)
# Для парсинга комментариев.
python_style_comment = originalTextFor(Literal('#') + restOfLine)
comments = (cppStyleComment |
python_style_comment).setParseAction(
self._create_comment_list
)
# Для парсинга директивы include.
include_line = (Optional(action_symbols, default='')('action')
+ Keyword('include')
+ Word(printables, excludeChars=';{}')
+ Optional(semicolon.suppress())
).setParseAction(
self._add_include_line
)
# Для парсинга простых директив состоящих из одного
# или двух параметров.
plain_line = (Group(Optional(action_symbols, default='')('action')
+ statement)('name')
+ Optional(parameter_value)
+ Optional(semicolon.suppress())).setParseAction(
self._add_plain_line
)
# Метод для парсинга IP адресов.
ip_line = (Group(Optional(action_symbols, default='')('action')
+ ip_value
+ Optional(semicolon.suppress()))('IP')
).setParseAction(self._add_ipline)
# Парсеры параметров.
param_line = (include_line |
ip_line |
plain_line)
# Парсер блока параметров.
param_block = (Group(Optional(action_symbols, default='')('action')
+ statement + Optional(statement_name)
+ Optional(statement_class))('name')
+ block('content')
+ Optional(semicolon.suppress())).setParseAction(
self._add_param_block
)
# Виды блочных директив.
block_types = (inet_spec | param_block)
# Парсер параметров с комментариями.
# Note: Применение parser.ignore(comments) является причиной странного
# поведения парсера, при котором невозможно многократное повторное
# применение формата после установки флага ignore_comments.
if self._ignore_comments:
param_line_with_comments = (ZeroOrMore(comments).suppress()(
'comments'
)
+ param_line('value')
).setParseAction(
self._add_comments_to_paramline
)
else:
param_line_with_comments = (ZeroOrMore(comments)('comments')
+ param_line('value')
).setParseAction(
self._add_comments_to_paramline
)
# Парсер блока с комментариями.
if self._ignore_comments:
param_block_with_comments = (ZeroOrMore(comments).suppress()(
'comments'
)
+ block_types('value')
).setParseAction(
self._add_comments_to_block
)
else:
param_block_with_comments = (ZeroOrMore(comments)('comments')
+ block_types('value')
).setParseAction(
self._add_comments_to_block
)
# Парсер содержимого блоков.
block_item = (param_block_with_comments |
param_line_with_comments)
# Для парсинга всего блока с любым содержимым.
block << Group(left_brace.suppress() + ZeroOrMore(block_item)
+ right_brace.suppress())
# Парсер всего документа.
self._document_parser = OneOrMore(block_item)
def _parse_text(self, text):
parsing_result = self._document_parser.parseString(text, parseAll=True)
list_of_elements = parsing_result.asList()
for part in list_of_elements:
self._join_dictionary(self._document_dictionary,
part)
def _join_dictionary(self, out_dictionary, dictionary_to_add):
for key in dictionary_to_add:
if dictionary_to_add == OrderedDict():
return
if key in out_dictionary and\
isinstance(dictionary_to_add[key], OrderedDict) and\
isinstance(out_dictionary[key], OrderedDict):
self._join_dictionary(out_dictionary[key],
dictionary_to_add[key])
else:
out_dictionary[key] = dictionary_to_add[key]
def _add_plain_line(self, current_parse):
name = tuple(current_parse.name.asList())
if not current_parse.parameter == '':
value = current_parse.parameter
else:
if current_parse.name.action == '-':
return OrderedDict()
value = ''
return OrderedDict({name: [value]})
def _add_include_line(self, current_parse):
name = current_parse.asList()
return OrderedDict({tuple(name): ['']})
def _add_ipline(self, current_parse):
ip_value = current_parse.IP
return OrderedDict({tuple(ip_value): ['']})
def _add_inet_specline(self, current_parse):
# Удаляем пробельные символы из второго параметра директивы.
current_parse.name.parameter = current_parse.name.parameter.strip()
block_name = tuple(current_parse.name.asList())
block_content = current_parse.content.asList()
content = OrderedDict({'#': []})
for item in block_content:
current_keyword, values = item
current_keyword = tuple(current_keyword)
content[current_keyword] = values
return OrderedDict({block_name: content})
def _add_param_block(self, current_parse):
block_name = tuple(current_parse.name.asList())
block_content = current_parse.content.asList()
content = OrderedDict({'#': []})
for item in block_content:
self._join_dictionary(content, item)
return OrderedDict({block_name: content})
def _add_comments_to_paramline(self, current_parse):
[parameter] = current_parse.value.asList()
comments = current_parse.comments
if parameter == OrderedDict():
if not comments == '':
self._last_comments_list.extend(comments.asList())
return OrderedDict()
name = next(iter(parameter))
value = parameter[name]
if not comments == '':
comments_list = comments.asList()
parameter[name] = (self._last_comments_list
+ comments_list + value)
self._last_comments_list = []
return parameter
def _add_comments_to_block(self, current_parse):
[value] = current_parse.value
[block_name] = value
block = value[block_name]
if not current_parse.comments == '':
block['#'] = (self._last_comments_list
+ current_parse.comments.asList())
self._last_comments_list = []
else:
block.pop('#')
return value
def _create_comment_list(self, current_parse):
comments_list = []
comments = current_parse.asList()
for comment in comments:
lines = comment.splitlines()
for line in lines:
comments_list.append(line.strip())
return comments_list

150
calculate/templates/format/compiz_format.py

@ -0,0 +1,150 @@
# vim: fileencoding=utf-8
#
from .base_format import BaseFormat
from collections import OrderedDict
from pyparsing import originalTextFor, Literal, ZeroOrMore, Word, printables,\
OneOrMore, alphanums, ParseException, restOfLine,\
pyparsing_unicode, Group, Optional
class CompizFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = [self._parse_comment_line,
self._parse_section_line,
self._parse_parameter_line,
self._parse_to_delete_line]
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._comments_processing = True
self._need_finish = True
self._format = 'compiz'
self._current_section = OrderedDict()
self._current_section_name = ''
self._last_comments_list = []
self._initialize_parser()
if document_text == '':
self._document_dictionary = OrderedDict()
else:
document_lines = self._get_list_of_logic_lines(document_text)
self._lines_to_dictionary(document_lines)
def _initialize_parser(self):
section_name = originalTextFor(
OneOrMore(Word(alphanums+'_'))
)
action_symbols = (Literal('!') | Literal('-'))
self._section_line = (Literal('[').suppress()
+ Optional(action_symbols, default='')('action')
+ section_name('name')
+ Literal(']').suppress())('section_name')
parameter_name = originalTextFor(
OneOrMore(Word(printables,
excludeChars='='))
)
parameter_value = Word(printables)
self._parameter_line = (Group(Optional(action_symbols,
default='')('action')
+ parameter_name('name')
)('parameter_name')
+ Literal('=').suppress()
+ parameter_value('parameter_value'))
self._parameter_to_delete = (action_symbols('action')
+ parameter_name('name')
+ restOfLine.suppress())('parameter_name')
self._comment_line = originalTextFor(
Literal('#')
+ ZeroOrMore(Word(printables
+ pyparsing_unicode.alphanums))
)('comment')
def _parse_section_line(self, line):
try:
self._item_to_add = OrderedDict()
parsing_result = self._section_line.parseString(line)
self._match = True
if self._current_section_name != '':
if self._current_section_name in \
self._document_dictionary.keys():
self._document_dictionary[self._current_section_name].\
update(self._current_section)
self._current_section = OrderedDict()
else:
self._item_to_add[self._current_section_name] = \
self._current_section
self._current_section = OrderedDict()
self._ready_to_update = True
self._current_section_name = tuple(
parsing_result.section_name.asList()
)
if self._last_comments_list != []:
self._current_section['#'] = self._last_comments_list
self._last_comments_list = []
except ParseException:
return
def _parse_parameter_line(self, line):
try:
parsing_result = self._parameter_line.parseString(line)
self._match = True
key_value = tuple(parsing_result.parameter_name.asList())
parameter_value = parsing_result.parameter_value
parameter_value = self._last_comments_list + [parameter_value]
self._last_comments_list = []
self._current_section[key_value] = parameter_value
except ParseException:
return
def _parse_to_delete_line(self, line):
try:
parsing_result = self._parameter_to_delete.parseString(line)
self._match = True
if parsing_result.action == '-':
return
key_value = tuple(parsing_result.parameter_name.asList())
parameter_value = self._last_comments_list
self._last_comments_list = []
self._current_section[key_value] = parameter_value
except ParseException:
return
def _parse_comment_line(self, line):
try:
parsing_result = self._comment_line.parseString(line)
self._match = True
if not self._ignore_comments:
self._last_comments_list.append(parsing_result.comment)
except ParseException:
return
def _finish_method(self):
if self._current_section_name in self._document_dictionary.keys():
self._document_dictionary[self._current_section_name].update(
self._current_section
)
else:
self._item_to_add[self._current_section_name] =\
self._current_section
self._document_dictionary.update(self._item_to_add)
self._current_section = OrderedDict()

62
calculate/templates/format/diff_format.py

@ -0,0 +1,62 @@
# vim: fileencoding=utf-8
#
from calculate.utils.files import Process
from os import path
class DiffFormat():
def __init__(self, document_text: str):
self._patch_text = document_text
self._root_path = ''
self._last_level = 0
# вынести в более общий класс или куда-то еще.
self._changed_files_list = []
def execute_format(self, root_path):
if path.exists(root_path):
self._root_path = root_path
else:
# Какая-то обработка ошибки.
error_message = 'Root path does not exist.'
print(error_message)
return False
if self._patch_text:
return self._patch_document()
else:
# Какая-то обработка ошибки.
error_message = 'Empty patch file.'
print(error_message)
return False
def _patch_document(self):
for level in range(0, 4):
patch_dry_run = Process('patch', '--dry-run',
'-p{}'.format(level), cwd=self._root_path)
patch_dry_run.write(self._patch_text)
if patch_dry_run.success():
break
patch_dry_run = Process('patch', '-R', '--dry-run',
'-p{}'.format(level), cwd=self._root_path)
patch_dry_run.write(self._patch_text)
if patch_dry_run.success():
return ''
else:
# Какая-то обработка ошибки.
error_message = 'Correction failed.'
print(error_message)
return False
self._last_level = level
patch_run = Process('patch', '-p{}'.format(level), cwd=self._root_path)
patch_run.write(self._patch_text)
if patch_run.success():
for line in patch_run:
if line.startswith('patching file'):
self._changed_files_list.append(line[13:].strip())
return patch_run.read()
else:
return ''

229
calculate/templates/format/dovecot_format.py

@ -0,0 +1,229 @@
# vim: fileencoding=utf-8
#
# ToDo: написать счетчик скобок для финальной оценки корректности
# документа.
#
from .base_format import BaseFormat
from collections import OrderedDict
from pyparsing import originalTextFor, Literal, ZeroOrMore, Word, printables,\
OneOrMore, alphanums, ParseException, pyparsing_unicode,\
Group, Optional, alphas, lineEnd, lineStart, Keyword
class DovecotFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = [self._parse_comment_line,
self._parse_section_start_line,
self._parse_include_line,
self._parse_section_end_line,
self._parse_parameter_line,
self._parse_parameter_to_delete_line]
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._need_finish = True
self._comments_processing = True
self._format = 'dovecot'
self._section_stack = OrderedDict()
self._current_section_name = ''
self._last_comments_list = []
self._initialize_parser()
if document_text == '':
self._document_dictionary = OrderedDict()
else:
document_lines = self._get_list_of_logic_lines(document_text)
self._lines_to_dictionary(document_lines)
def _initialize_parser(self):
# Знаки пунктуации и действий.
left_brace = Literal('{')
right_brace = Literal('}')
action_symbols = (Literal('!') | Literal('-'))
self._comment_line_parser = originalTextFor(
Literal('#')
+ ZeroOrMore(Word(
printables
+ pyparsing_unicode.alphanums)
)
)('comment')
# Для парсинга строк с началом секций.
section = Word(alphas, alphanums+'-_', excludeChars='{}')
section_name = Word(printables, excludeChars='{}')
self._section_start_parser = Group(Optional(action_symbols,
default='')('action')
+ section
+ Optional(section_name)
+ left_brace.suppress())('name')
# Для парсинга строк, указывающих конец секций.
self._section_end_parser = lineStart() + right_brace + lineEnd()
# Для парсинга строк, содержащих параметры.
parameter_name = Word(alphas, alphanums+'_-', excludeChars='{}=')
parameter_value = OneOrMore(Word(printables))
self._parameter_line_parser = (Group(Optional(action_symbols,
default='')('action')
+ parameter_name)('name')
+ Literal('=').suppress()
+ originalTextFor(
parameter_value
)('value'))
# Для парсинга строк с параметрами, подлежащими удалению.
self._parameter_to_delete_parser = (action_symbols('action')
+ parameter_name
+ Optional(Literal('=')).suppress()
)
# Для парсинга строк, содержащих директиву !include.
include = Keyword('!include') | Keyword('!include_try')
include_line_plain = (Optional(~action_symbols, default='')('action')
+ include('keyword') + Word(printables)('value'))
include_line_to_delete = (action_symbols('action') + include('keyword')
+ Word(printables)('value'))
self._include_line_parser = (include_line_plain |
include_line_to_delete)
def _parse_section_start_line(self, line):
try:
parsing_result = self._section_start_parser.parseString(line)
self._match = True
section_name = tuple(parsing_result.name.asList())
if not self._last_comments_list == []:
section_content = OrderedDict({'#': self._last_comments_list})
self._last_comments_list = []
else:
section_content = OrderedDict()
new_section = OrderedDict({section_name: section_content})
if self._section_stack == OrderedDict():
if section_name in self._document_dictionary:
new_section = OrderedDict(
{section_name:
self._document_dictionary[section_name]}
)
else:
if section_name in \
self._section_stack[self._current_section_name]:
new_section = OrderedDict({
section_name:
self._section_stack[self._current_section_name]
[section_name]
})
else:
self._section_stack[self._current_section_name].update(
new_section
)
self._section_stack.update(new_section)
self._current_section_name = section_name
except ParseException:
return
def _parse_section_end_line(self, line):
try:
self._section_end_parser.parseString(line)
self._match = True
last_section_name, last_section = self._section_stack.popitem()
if self._section_stack == OrderedDict():
self._item_to_add = OrderedDict({last_section_name:
last_section})
self._ready_to_update = True
self._current_section_name = ''
else:
self._current_section_name = next(reversed(
self._section_stack
))
except ParseException:
return
def _parse_parameter_line(self, line):
try:
parsing_result = self._parameter_line_parser.parseString(line)
self._match = True
parameter_name = tuple(parsing_result.name.asList())
parameter_value = (self._last_comments_list
+ [parsing_result.value.strip()])
self._last_comments_list = []
parameter = OrderedDict({parameter_name: parameter_value})
if self._section_stack == OrderedDict():
self._item_to_add = parameter
self._ready_to_update = True
else:
self._section_stack[self._current_section_name].update(
parameter
)
except ParseException:
return
def _parse_parameter_to_delete_line(self, line):
try:
parsing_result = self._parameter_to_delete_parser.parseString(line)
self._match = True
parameter_name = tuple(parsing_result.asList())
parameter_value = self._last_comments_list + ['']
self._last_comments_list = []
parameter = OrderedDict({parameter_name: parameter_value})
if self._section_stack == OrderedDict():
self._item_to_add = parameter
self._ready_to_update = True
else:
self._section_stack[self._current_section_name].update(
parameter
)
except ParseException:
return
def _parse_include_line(self, line):
try:
parsing_result = self._include_line_parser.parseString(line)
self._match = True
parameter_name = tuple(parsing_result.asList())
if parsing_result.action == '-':
return
parameter_value = self._last_comments_list + ['']
self._last_comments_list = []
include_item = OrderedDict({parameter_name: parameter_value})
if self._section_stack == OrderedDict():
self._item_to_add = include_item
self._ready_to_update = True
else:
self._section_stack[self._current_section_name].update(
include_item
)
except ParseException:
return
def _parse_comment_line(self, line):
try:
parsing_result = self._comment_line_parser.parseString(line)
self._match = True
if not self._ignore_comments:
self._last_comments_list.append(parsing_result.comment)
except ParseException:
return

26
calculate/templates/format/json_format.py

@ -0,0 +1,26 @@
# vim: fileencoding=utf-8
#
from .base_format import BaseFormat
from collections import OrderedDict
import json
class JSONFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = []
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._format = 'json'
if document_text == '':
self._document_dictionary = OrderedDict()
else:
self._text_to_dictionary(document_text)
def _text_to_dictionary(self, json_file_text):
self._document_dictionary = json.loads(json_file_text,
object_pairs_hook=OrderedDict)
def get_document_text(self):
json_file_text = json.dumps(self._document_dictionary, indent=4)
return json_file_text

155
calculate/templates/format/kde_format.py

@ -0,0 +1,155 @@
# vim: fileencoding=utf-8
#
from .base_format import BaseFormat
from collections import OrderedDict
from pyparsing import originalTextFor, Literal, ZeroOrMore, Word, printables,\
OneOrMore, alphanums, ParseException, restOfLine,\
pyparsing_unicode, Group, Optional
class KDEFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = [self._parse_comment_line,
self._parse_section_line,
self._parse_parameter_line,
self._parse_to_delete_line]
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._comments_processing = True
self._need_finish = True
self._format = 'kde'
self._current_section = OrderedDict()
self._current_section_name = ''
self._last_comments_list = []
self._initialize_parser()
if document_text == '':
self._document_dictionary = OrderedDict()
else:
document_lines = self._get_list_of_logic_lines(document_text)
self._lines_to_dictionary(document_lines)
def _initialize_parser(self):
action_symbols = (Literal('!') | Literal('-'))
section_name_part_content = originalTextFor((OneOrMore(
Word(alphanums+':'))))
section_name_part = (Literal('[').suppress()
+ section_name_part_content
+ Literal(']').suppress())
self._section_line = (Literal('[').suppress()
+ Optional(action_symbols, default='')('action')
+ section_name_part_content
+ Literal(']').suppress()
+ ZeroOrMore(section_name_part))('section_name')
parameter_name = originalTextFor(
OneOrMore(Word(printables,
excludeChars='='))
)('parameter_name')
parameter_value = originalTextFor(OneOrMore(Word(
pyparsing_unicode.alphanums
+ printables))
)('parameter_value')
self._parameter_line = (Group(Optional(action_symbols,
default='')('action')
+ parameter_name)('parameter_name')
+ Literal('=').suppress()
+ parameter_value('parameter_value'))
self._parameter_to_delete = (action_symbols('action')
+ parameter_name('name')
+ restOfLine.suppress())('parameter_name')
self._comment_line = originalTextFor(
Literal('#')
+ ZeroOrMore(Word(printables
+ pyparsing_unicode.alphanums))
)('comment')
def _parse_section_line(self, line):
try:
self._item_to_add = OrderedDict()
parsing_result = self._section_line.parseString(line)
self._match = True
if self._current_section_name != '':
if self._current_section_name in \
self._document_dictionary.keys():
self._document_dictionary[self._current_section_name].\
update(self._current_section)
self._current_section = OrderedDict()
else:
self._item_to_add[self._current_section_name] = \
self._current_section
self._current_section = OrderedDict()
self._ready_to_update = True
self._current_section_name = tuple(
parsing_result.section_name.asList()
)
if self._last_comments_list != []:
self._current_section['#'] = self._last_comments_list
self._last_comments_list = []
except ParseException:
return
def _parse_parameter_line(self, line):
try:
parsing_result = self._parameter_line.parseString(line)
self._match = True
key_value = tuple(parsing_result.parameter_name.asList())
parameter_value = parsing_result.parameter_value
parameter_value = self._last_comments_list + [parameter_value]
self._last_comments_list = []
self._current_section[key_value] = parameter_value
except ParseException:
return
def _parse_to_delete_line(self, line):
try:
parsing_result = self._parameter_to_delete.parseString(line)
self._match = True
if parsing_result.action == '-':
return
key_value = tuple(parsing_result.parameter_name)
parameter_value = self._last_comments_list
self._last_comments_list = []
self._current_section[key_value] = parameter_value
except ParseException:
return
def _parse_comment_line(self, line):
try:
parsing_result = self._comment_line.parseString(line)
self._match = True
if not self._ignore_comments:
self._last_comments_list.append(parsing_result.comment)
except ParseException:
return
def _finish_method(self):
if self._current_section_name in self._document_dictionary.keys():
self._document_dictionary[self._current_section_name].update(
self._current_section
)
else:
self._item_to_add[self._current_section_name] =\
self._current_section
self._document_dictionary.update(self._item_to_add)
self._current_section = OrderedDict()

100
calculate/templates/format/kernel_format.py

@ -0,0 +1,100 @@
# vim: fileencoding=utf-8
#
from .base_format import BaseFormat
from collections import OrderedDict
from pyparsing import Word, Literal, alphanums, printables, originalTextFor,\
ZeroOrMore, OneOrMore, ParseException, restOfLine,\
pyparsing_unicode, Group, Optional
class KernelFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = [self._parse_comment_line,
self._parse_parameter_line,
self._parse_to_delete_line]
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._comments_processing = True
self._format = 'kernel'
self._last_comments_list = []
self._initialize_parser()
if document_text == '':
self._document_dictionary = OrderedDict()
else:
documentLines = self._get_list_of_logic_lines(document_text)
self._lines_to_dictionary(documentLines)
def _initialize_parser(self):
parameter_name = Word(alphanums+'_')('parameter_name')
parameter_value = originalTextFor(
OneOrMore(Word(printables))
)('parameter_value')
action_symbols = (Literal('!') | Literal('-'))
self._parameter_line = (Group(Optional(action_symbols,
default='')('action')
+ parameter_name('name')
)('parameter_name')
+ Literal('=').suppress()
+ parameter_value('parameter_value'))
self._parameter_to_delete = (Group(action_symbols('action')
+ parameter_name('name')
)('parameter_name')
+ restOfLine.suppress())
self._comment_line = originalTextFor(
Literal('#').suppress()
+ ZeroOrMore(Word(printables
+ pyparsing_unicode.alphanums))
)('Comment')
def _parse_parameter_line(self, line):
try:
self._item_to_add = OrderedDict()
parsing_result = self._parameter_line.parseString(line)
key_value = tuple(parsing_result.parameter_name.asList())
parameter_value = parsing_result.parameter_value
parameter_value = self._last_comments_list + [parameter_value]
self._last_comments_list = []
self._item_to_add[key_value] = parameter_value
self._ready_to_update = True
self._match = True
except ParseException:
return
def _parse_to_delete_line(self, line):
try:
self._item_to_add = OrderedDict()
parsing_result = self._parameter_to_delete.parseString(line)
if parsing_result.action == '-':
return
key_value = (parsing_result.parameter_name.action,
parsing_result.parameter_name.name.upper())
parameter_value = self._last_comments_list
self._last_comments_list = []
self._item_to_add[key_value] = parameter_value
self._ready_to_update = True
self._match = True
except ParseException:
return
def _parse_comment_line(self, line):
try:
result = self._comment_line.parseString(line)
self._match = True
if not self._ignore_comments:
self._last_comments_list.append(result.Comment)
except ParseException:
return

522
calculate/templates/format/ldap_format.py

@ -0,0 +1,522 @@
# vim: fileencoding=utf-8
#
from .base_format import BaseFormat
from collections import OrderedDict
from pyparsing import originalTextFor, Literal, ZeroOrMore, Word, printables,\
OneOrMore, alphanums, ParseException, restOfLine,\
pyparsing_unicode, nums, delimitedList, Optional,\
Keyword, SkipTo, Group
class LDAPFormat(BaseFormat):
def __init__(self, document_text: str, ignore_comments=False):
processing_methods = [self._parse_comment_line,
self._parse_type_line,
self._parse_access_line,
self._parse_access_line_to_delete,
self._parse_syncrepl_line,
self._parse_syncrepl_line_to_delete,
self._parse_notunique_line,
self._parse_index_line,
self._parse_index_line_to_delete,
self._parse_plain_directive_line,
self._parse_plain_directive_line_to_delete]
super().__init__(processing_methods)
self._ignore_comments = ignore_comments
self._comments_processing = True
self._need_finish = True
self._format = 'ldap'
if self._ignore_comments:
self._current_type_section = OrderedDict()
else:
self._current_type_section = OrderedDict({'#': []})
self._current_type = ('', 'global')
self._last_comments_list = []
self._initialize_parser()
if document_text == '':
self._document_dictionary = OrderedDict()
else:
document_lines = self._get_list_of_logic_lines(document_text)
self._lines_to_dictionary(document_lines)
def _initialize_parser(self):
self._comment_line = originalTextFor(
Literal('#')
+ ZeroOrMore(Word(printables
+ pyparsing_unicode.alphanums))
)('comment')
action_symbols = (Literal('!') | Literal('-'))
assignment = Literal('=')
# Для парсинга строк c директивами неуникальными для секции.
not_unique_directives = originalTextFor(
Keyword('include') |
Keyword('moduleload')
)
not_unique_value = originalTextFor(
OneOrMore(Word(printables))
)('value')
self._not_unique_parser = (Optional(action_symbols,
default='')('action')
+ not_unique_directives
+ not_unique_value + restOfLine.suppress())
# Для выделения областей global, backend и database.
type_sections_keywords = originalTextFor(
Keyword('backend') |
Keyword('database')
)
type_value = originalTextFor(Word(alphanums))
self._type_line = (Optional(action_symbols, default='')('action')
+ type_sections_keywords
+ type_value
+ restOfLine.suppress())
# Для парсинга конструкции syncrepl rid=<replica ID> <parameters>
content_without_spaces = Word(printables, excludeChars='"')
content_with_spaces = (Literal('"')
+ OneOrMore(Word(printables,
excludeChars='"'))
+ Literal('"'))
parameter_without_spaces = (Word(printables, excludeChars='"=')
+ assignment.suppress()
+ content_without_spaces)
parameter_with_spaces = (Word(printables, excludeChars='"=')
+ assignment.suppress()
+ content_with_spaces)
values = OneOrMore(originalTextFor(parameter_with_spaces |
parameter_without_spaces))('Values')
syncrepl_replica_id = originalTextFor(Literal('rid')
+ assignment.suppress()
+ Word(nums))('replicaID')
self._syncrepl_line_parser = (Group(Optional(action_symbols,
default='')('action')
+ Keyword('syncrepl')
+ syncrepl_replica_id)('name')
+ values('Values')
+ restOfLine.suppress())
self._syncrepl_value_parser = (Group(Optional(action_symbols,
default='')('action')
+ originalTextFor(
Word(
printables,
excludeChars='"='
)
))('name')
+ assignment.suppress()
+ originalTextFor(
OneOrMore(
Word(printables)
)
)('value'))
self._syncrepl_line_to_delete_parser = (Group(Optional(
action_symbols,
default=''
)('action')
+ Keyword('syncrepl')
+ syncrepl_replica_id)('name')
+ restOfLine.suppress())
# Для парсинга конструкции
# access to <what> by <who>|<access level>|<control>
access_keyword = originalTextFor(Literal('access to'))('keyword')
value = originalTextFor(parameter_with_spaces |
parameter_without_spaces |
content_without_spaces |
content_with_spaces)
self._access_line_parser = (Group(Optional(action_symbols,
default='')('action')
+ access_keyword
+ value)('name')
+ Keyword('by').suppress()
+ delimitedList(
originalTextFor(value +
SkipTo(
Keyword('by'),
include=False) |
restOfLine
),
delim='by'
)('Values'))
self._access_value_parser = (Group(Optional(action_symbols,
default='')('action')
+ originalTextFor(value))('name')
+ originalTextFor(
Optional(Word(alphanums))
)('value'))
self._access_line_to_delete_parser = (Group(action_symbols('action')
+ access_keyword
+ value
+ restOfLine.suppress())('name'))
# Для парсинга строк с директивами index.
self._index_line_parser = (Group(Optional(action_symbols,
default='')('action')
+ Keyword('index')
+ originalTextFor(Word(printables))
)('name')
+ originalTextFor(
OneOrMore(Word(printables))
)('value'))
self._index_line_to_delete_parser = (Group(action_symbols('action')
+ Keyword('index')
+ originalTextFor(
Word(printables)
))('name'))
# Для парсинга остальных директив.
self._directive_line_parser = (Group(Optional(action_symbols,
default='')('action')
+ originalTextFor(
Word(printables)
))('name')
+ originalTextFor(
OneOrMore(Word(
printables
)
))('value'))
self._directive_line_to_delete_parser = (action_symbols('action')
+ originalTextFor(
Word(printables)
))('name')
def _get_list_of_logic_lines(self, text):
list_of_lines = []
lines_to_join = []
for line in text.splitlines():
if line.strip() == '':
continue
if not line.startswith(' ') and not line.startswith('\t'):
joined_line = "".join(lines_to_join)
if joined_line != '':
list_of_lines.append(joined_line)
lines_to_join = []
line.strip()
else:
line = ' ' + line.strip()
lines_to_join.append(line)
joined_line = "".join(lines_to_join)
list_of_lines.append(joined_line)
return list_of_lines
def _parse_type_line(self, line):
try:
self._item_to_add = OrderedDict()
parsing_result = self._type_line.parseString(line)
self._match = True
type_name = tuple(parsing_result.asList())
if self._current_type in self._document_dictionary.keys():
self._document_dictionary[self._current_type].update(
self._current_type_section
)
else:
self._item_to_add[self._current_type] =\
self._current_type_section
self._ready_to_update = True
# Если глобальная область пуста -- передаем ее комментарии
# следующей за ней области.
if self._current_type == ('', 'global') and\
list(self._current_type_section.keys()) == ['#']:
self._last_comments_list = self._current_type_section['#']
self._item_to_add[self._current_type] = OrderedDict()
self._current_type_section = OrderedDict()
self._current_type = type_name