You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

604 lines
26 KiB

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

import re
import sys
import os
import importlib
import importlib.util
import site
from calculate.variables.old_vars.datavars import Variable, HashVariable,\
TableVariable
from calculate.variables.datavars import NamespaceNode, VariableNode,\
ListType, IntegerType,\
FloatType, IniType, TableType,\
Namespace, VariableError
from calculate.utils.gentoo import ProfileWalker
from calculate.utils.files import list_directory, read_file, FilesError
from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
empty, printables, OneOrMore, lineno, line, col, SkipTo,\
LineEnd, Combine, nums
from enum import Enum
from contextlib import contextmanager
class Define(Enum):
assign = 0
append = 1
remove = 2
class CalculateIniParser:
'''Класс парсера calculate.ini файлов.'''
def __init__(self):
self._errors = []
self.operations = {"=": Define.assign,
"+=": Define.append,
"-=": Define.remove}
lbrack = Literal("[")
rbrack = Literal("]")
# comma = Literal(",").suppress()
comment_symbol = Literal(';') | Literal('#')
# Define = self.Define
value_operation = (Literal("=") | Combine(Literal("+") + Literal("="))
| Combine(Literal("-") + Literal("=")))
comment = comment_symbol + Optional(restOfLine)
section_name = (lbrack.suppress() + (~Word(nums)
+ Word(printables+'\t',
excludeChars='[]'))
+ rbrack.suppress())
value_name = Word(printables+'\t', excludeChars='=-+')
# non_comma = Word(printables+'\t', excludeChars=',')
clear_section = lbrack.suppress() + Group(empty) + rbrack.suppress()
row_index = lbrack.suppress() + Word(nums) + rbrack.suppress()
namespace_start = Group(OneOrMore(section_name)
+ (clear_section | ~lbrack)
+ LineEnd().suppress())
table_start = Group(OneOrMore(section_name)
+ (row_index | clear_section | ~lbrack)
+ LineEnd().suppress())
section_start = (namespace_start('namespace') |
table_start('table'))
# Если содержимое ini-файла не предваряется заголовком секции,
# значит эта строка ошибочна.
unexpected = Group(~section_start + SkipTo(LineEnd(),
include=True))("error")
unexpected.setParseAction(self._unexpected_token)
key_value = (~lbrack + value_name
+ value_operation + empty
+ restOfLine + LineEnd().suppress())
def strip_key_value(tokens):
tokens[0] = tokens[0].strip()
tokens[1] = tokens[1].strip()
key_value.setParseAction(strip_key_value)
self.ini_section_parser = (section_start
+ Group(ZeroOrMore(
Group(key_value | unexpected)))
| unexpected)
self.ini_section_parser.ignore(comment)
def parse(self, data: str):
for tokens, start, end in self.ini_section_parser.scanString(data):
if tokens.getName() == "error":
continue
section, defkeys = tokens
if section.getName() == 'namespace':
section_list = section.asList()
if section_list[-1] == []:
yield {'clear_section': (section_list[:-1], )}
else:
yield {'start_section': (section_list, )}
for defkey in defkeys:
if defkey.getName() == "error":
continue
yield {'define_key': (defkey[0], defkey[2],
self.operations[defkey[1]])}
else:
table_list = section.asList()
if table_list[-1] == []:
yield {'clear_table': (table_list[:-1], )}
else:
table_values = {}
for defkey in defkeys:
if defkey.getName() == "error":
continue
table_values.update({defkey[0]: defkey[2]})
yield {'start_table': (table_list, table_values)}
def _unexpected_token(self, string, location, tokens):
'''Метод вызываемый парсером, если обнаружена некорректная строка,
предназначен для получения некорректной строки и ее дальнейшего
разбора.'''
error_line = line(location, string).strip()
if error_line:
self._errors.append((error_line, lineno(location, string),
col(location, string)))
@property
def errors(self):
errors = self._errors
self._errors = []
return errors
class NamespaceIniFiller:
'''Класс, предназначенный для наполнения Namespace объекта переменными
из calculate.ini файла.'''
available_sections = {'custom'}
def __init__(self, restrict_creation=True):
self.ini_parser = CalculateIniParser()
self.restricted = restrict_creation
self.modify_only = False
def error(self, lineno, error_message):
self.errors.append(lineno, error_message)
def fill(self, namespace: NamespaceNode, ini_file_text: str) -> None:
'''Метод для разбора calculate.ini файла и добавления всех его
переменных в указанное пространство имен.'''
self.namespace = namespace
self.current_namespace = self.namespace
self.errors = []
for parsed_line in self.ini_parser.parse(ini_file_text):
self._line_processor(**parsed_line)
def _line_processor(self, start_section=None,
clear_section=None,
start_table=None,
clear_table=None,
define_key=None,
error=None, **kwargs):
if start_section is not None:
self.start_section(*start_section)
elif clear_section is not None:
self.clear_section(*clear_section)
elif start_table is not None:
self.start_table(*start_table)
elif clear_table is not None:
self.clear_table(*clear_table)
elif define_key is not None:
self.define_key(*define_key)
for error in self.ini_parser.errors:
self.set_error(*error)
def start_section(self, sections: str) -> None:
'''Метод для получения доступа и создания пространств имен.'''
print('SECTIONS: {}'.format(sections))
if self.restricted:
self.modify_only = sections[0] not in self.available_sections
self.current_namespace = self.namespace
for section in sections:
print('START SECTION: {}'.format(section))
if isinstance(self.current_namespace, Datavars):
if section not in self.current_namespace:
raise VariableError("variables package '{}' is not found".
format(section))
elif isinstance(self.current_namespace, NamespaceNode):
if section not in self.current_namespace.namespaces:
if not self.modify_only:
self.current_namespace.add_namespace(
NamespaceNode(section))
else:
self.current_namespace = None
return
self.current_namespace = self.current_namespace.namespaces[section]
def clear_section(self, sections: list) -> None:
'''Метод для очистки пространства имен.'''
if self.restricted:
self.modify_only = sections[0] not in self.available_sections
current_namespace = self.namespace
for section in sections:
if (isinstance(current_namespace, Datavars) and
section in current_namespace):
current_namespace = current_namespace[section]
elif isinstance(current_namespace, NamespaceNode):
if section in current_namespace.namespaces:
current_namespace = current_namespace[section]
elif (section in current_namespace.variables and
current_namespace.variables[section].variable_type
is TableType):
table_variable = current_namespace.variables[section]
table_to_clear = table_variable.get_value()
table_to_clear.clear()
table_variable.source = table_to_clear
return
else:
return
if not self.modify_only:
current_namespace.clear()
def start_table(self, sections: str, row) -> None:
'''Метод для создания и модификации таблиц.'''
if self.restricted:
self.modify_only = sections[0] not in self.available_sections
self.current_namespace = self.namespace
row_index = int(sections.pop())
table_name = sections.pop()
for section in sections:
if section not in self.current_namespace.namespaces:
if not self.modify_only:
self.current_namespace.add_namespace(
NamespaceNode(section))
else:
self.current_namespace = None
return
self.current_namespace = self.current_namespace.namespaces[section]
if table_name not in self.current_namespace.variables:
if not self.modify_only:
table_variable = VariableNode(table_name,
self.current_namespace,
variable_type=TableType,
source=[row])
else:
table_variable = self.current_namespace.variables[table_name]
table = table_variable.get_value()
if row_index < len(table):
table.change_row(row, row_index)
else:
table.add_row(row)
table_variable.source = table
def define_key(self, key: str, value: str, optype) -> None:
print('DEFINE KEY: {} -> {} TO {}'.format(key, value,
self.current_namespace))
if self.current_namespace is None:
return
if optype == Define.assign:
if key not in self.current_namespace:
self.define_variable(key, value)
else:
self.change_value(key, value)
elif optype == Define.append:
if key not in self.current_namespace:
self.define_variable(key, value)
else:
self.append_value(key, value)
elif optype == Define.remove:
if key not in self.current_namespace:
self.define_variable(key, value)
else:
self.remove_value(key, value)
def change_value(self, key: str, value: str) -> None:
'''Метод для изменения значения переменной.'''
variable = self.current_namespace[key]
variable.source = value
def define_variable(self, key: str, value: str) -> None:
'''Метод для создания переменных в calculate.ini файле.'''
if not self.modify_only:
VariableNode(key, self.current_namespace, variable_type=IniType,
source=value)
else:
# TODO Какая-то обработка ошибки.
pass
def append_value(self, key: str, value: str) -> None:
'''Метод выполняющий действия возложенные на оператор +=.'''
variable = self.current_namespace[key]
variable_value = variable.get_value()
if variable.variable_type is IniType:
value_list = value.split(',')
variable_list = variable_value.split(',')
for item in value_list:
if item not in variable_list:
variable_list.append(item.strip())
variable_value = ','.join(variable_list)
elif variable.variable_type is ListType:
value_list = value.split(',')
for item in value_list:
if item not in variable_value:
variable_value.append(item.strip())
elif variable.variable_type is IntegerType:
variable_value += int(value)
elif variable.variable_type is FloatType:
variable_value += float(value)
variable.source = variable_value
def remove_value(self, key: str, value: str) -> None:
'''Метод выполняющий действия возложенные на оператор -=.'''
variable = self.current_namespace[key]
variable_value = variable.get_value()
if variable.variable_type is IniType:
value_list = value.split(',')
variable_list = [item.strip() for item in
variable_value.split(',')]
for item in value_list:
if item in variable_list:
variable_list.remove(item.strip())
variable_value = ','.join(variable_list)
elif variable.variable_type is ListType:
value_list = value.split(',')
for item in value_list:
if item in variable_value:
variable_value.remove(item.strip())
elif variable.variable_type is IntegerType:
variable_value -= int(value)
elif variable.variable_type is FloatType:
variable_value -= float(value)
variable.source = variable_value
def set_error(self, line, lineno, col):
'''Метод для добавления ошибки в лог.'''
self.error(lineno, "Syntax error: {}".format(line))
class VariableLoader:
'''Класс загрузчика переменных из python-файлов и из ini-файлов.'''
ini_basename = "calculate.ini"
def __init__(self, datavars, variables_path, repository_map=None):
self.datavars = datavars
self.ini_filler = NamespaceIniFiller()
self.variables_path = variables_path
self.variables_package = '.'.join(variables_path.split('/'))
self.repository_map = repository_map
def load_variables_package(self, package_name: str) -> None:
'''Метод для загрузки пакетов с переменными.'''
directory_path = os.path.join(self.variables_path, package_name)
package = '{}.{}'.format(self.variables_package, package_name)
package_namespace = NamespaceNode(package_name)
self.datavars.root.add_namespace(package_namespace)
self._fill_from_package(package_namespace, directory_path, package)
def load_from_profile(self):
if self.repository_map != {}:
self.repository_map = (self.repository_map or
self._get_repository_map(self.datavars))
profile_path = self.datavars.os.gentoo.profile.path
self._fill_from_ini(profile_path)
def _fill_from_package(self, current_namespace: NamespaceNode,
directory_path: str, package: str) -> None:
'''Метод для зaполнения переменных из python-файла.'''
file_nodes = []
directory_nodes = []
# Просматриваем директорию
for node in os.scandir(directory_path):
if node.is_dir():
directory_nodes.append(node)
elif node.is_file() and node.name.endswith('.py'):
file_nodes.append(node)
# Сначала загружаем переменные из файлов.
for file_node in file_nodes:
file_name = file_node.name[:-3]
Namespace.set_current_namespace(current_namespace)
with self.test(file_name, current_namespace):
# importlib.import_module('{}.{}'.format(package, file_name))
spec = importlib.util.spec_from_file_location(
'{}.{}'.format(package, file_name),
file_node.path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# Обходим остальные директории.
for directory_node in directory_nodes:
namespace = NamespaceNode(directory_node.name)
current_namespace.add_namespace(namespace)
self._fill_from_package(namespace, directory_node.path,
'{}.{}'.format(package,
directory_node.name))
def _fill_from_ini(self, profile_path):
'''Метод для зaполнения переменных из ini-файла.'''
print('PROCESSING INI FROM PROFILE')
profile_walker = ProfileWalker(self.ini_basename,
self.repository_map)
for file_path in profile_walker.find(profile_path):
try:
print('PROCESS FILE: {}'.format(file_path))
ini_file_text = read_file(file_path)
self.ini_filler.fill(self.datavars, ini_file_text)
except FilesError:
# TODO продумать обработку ошибок.
pass
def _get_repository_map(self, datavars):
return {repo['name']: repo['path']
for repo in datavars.os.gentoo.repositories}
def fill_from_custom_ini(self, file_path: str):
'''Метод для заполнения переменных из конкретного указанного файла.'''
print('LOAD FROM INI: {}'.format(file_path))
if os.path.exists(file_path):
ini_file_text = read_file(file_path)
self.ini_filler.fill(self.datavars, ini_file_text)
@contextmanager
def test(self, file_name, namespace):
print('IMPORT: {}.{}'.format(namespace.get_fullname(), file_name))
try:
yield self
finally:
print('IMPORTED FROM: {}.{}'.format(namespace.get_fullname(),
file_name))
class Datavars:
'''Класс для хранения переменных и управления ими.'''
def __init__(self, variables_path='calculate/vars', repository_map=None):
self._variables_path = variables_path
self._available_packages = self._get_available_packages()
self.root = NamespaceNode('<root>')
self._loader = VariableLoader(self, self._variables_path,
repository_map=repository_map)
Namespace.reset()
Namespace.set_datavars(self)
self._loader.load_from_profile()
def reset(self):
'''Метод для сброса модуля переменных.'''
self.root.clear()
self.root = NamespaceNode('<root>')
self._available_packages.clear()
self._available_packages = self._get_available_packages()
Namespace.set_datavars(self)
def _get_available_packages(self) -> dict:
'''Метод для получения словаря с имеющимися пакетами переменных
и путями к ним.'''
available_packages = dict()
for file_name in os.listdir(self._variables_path):
if file_name.startswith('__'):
continue
file_path = os.path.join(self._variables_path, file_name)
if os.path.isdir(file_path):
available_packages.update({file_name: file_path})
return available_packages
def __getattr__(self, package_name: str):
'''Метод возвращает ноду пространства имен, соответствующего искомому
пакету.'''
print('getattr: {}'.format(package_name))
if package_name in self.root.namespaces:
return self.root[package_name]
elif package_name not in self._available_packages:
raise VariableError("variables package '{}' is not found".
format(package_name))
else:
self._loader.load_variables_package(package_name)
return self.root[package_name]
def __getitem__(self, package_name: str) -> None:
'''Метод возвращает ноду пространства имен, соответствующего искомому
пакету.'''
print('getitem: {}'.format(package_name))
if package_name in self.root:
return self.root[package_name]
elif package_name not in self._available_packages:
raise VariableError("variables package '{}' is not found".
format(package_name))
else:
self._loader.load_variables_package(package_name)
return self.root[package_name]
def __contains__(self, package_name):
if package_name in self.root.namespaces:
return True
elif package_name not in self._available_packages:
return False
else:
self._loader.load_variables_package(package_name)
return True
@property
def namespaces(self):
return self.root.namespaces
class OldVariableLoader:
'''Класс, используемый для загрузки переменных из python модуля.'''
re_upper = re.compile("(.)([A-Z])")
def _get_varlike_attrs(self, obj):
'''Метод для получения списка аттрибутов похожих на переменные.'''
for attrname in (x for x in dir(obj) if x[:1].isupper()):
yield self.re_upper.sub(r"\1_\2", attrname).lower(), \
getattr(obj, attrname)
def fill(self, namespace, dirpath, package):
'''Загрузить в namespace переменные из указанных модулей.'''
for fullfn in list_directory(dirpath, fullpath=True):
dn, fn = os.path.split(fullfn)
if os.path.isdir(fullfn):
newns = namespace.add_namespace(Namespace(fn))
self.fill(newns, fullfn, "{}.{}".format(package, fn))
elif fn.endswith(".py"):
module = self._load_module_source(package, fn, fullfn)
for varname, cls in self._get_varlike_attrs(module):
if Variable.is_implementation(cls):
namespace.add_variable(cls(varname))
elif HashVariable.is_implementation(cls) or \
TableVariable.is_implementation(cls) or \
Namespace.is_implementation(cls):
_newns = namespace.add_namespace(cls(varname,
namespace))
for _varname, _cls in self._get_varlike_attrs(cls):
if Variable.is_implementation(_cls):
_newns.add_variable(_cls(_varname))
@classmethod
def default(cls):
site_dirs = [os.path.normpath(x) for x in site.getsitepackages()]
for site_dir in site_dirs:
calculate_dir = os.path.join(site_dir, "calculate/vars")
if os.path.exists(calculate_dir):
return (calculate_dir, "calculate.vars")
def _load_module_source(self, package, name, path):
if name.startswith('calculate.vars.'):
full_name = name
else:
full_name = '.'.join([package, name])
if full_name in sys.modules:
return sys.modules[full_name]
spec = importlib.util.spec_from_file_location(full_name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules[full_name] = module
return module
class OldProfileFiller:
"""
Заполнитель значений переменных из файлов calculate.ini в профилях
"""
basename = "calculate.ini"
def get_repository_map(self, datavars):
return {repo['name']: repo['path']
for repo in datavars.os.gentoo.repositories}
def fill(self, datavars, profile_path):
ini_filler = NamespaceIniFiller()
profile_walker = ProfileWalker(self.basename,
self.get_repository_map(datavars))
for file_path in profile_walker.find(profile_path):
try:
ini_file_text = read_file(file_path)
ini_filler.fill(datavars, ini_file_text)
except FilesError:
pass