The NamespaceIniFiller implementation is almost done for the new version of variables.

packages
Иванов Денис 4 years ago
parent 224b646793
commit 8f646d38b3

@ -40,6 +40,16 @@ class VariableType:
return value
class IniType(VariableType):
'''Класс, соответствующий типу переменных созданных в calculate.ini файлах.
'''
name = 'ini'
@classmethod
def process_value(self, value, variable):
return value
class StringType(VariableType):
'''Класс, соответствующий типу переменных с строковым значением.'''
name = 'string'
@ -191,8 +201,8 @@ class Hash:
else:
raise VariableError(("'{key}' is not found in the hash"
" '{hash_name}'").format(
key=key,
hash_name=self.master_variable.fullname))
key=key,
hash_name=self.master_variable.get_fullname()))
def __getitem__(self, key: str) -> HashValue:
if key in self._values:
@ -200,8 +210,8 @@ class Hash:
else:
raise VariableError(("'{key}' is not found in the hash"
" '{hash_name}'").format(
key=key,
hash_name=self.master_variable.fullname))
key=key,
hash_name=self.master_variable.get_fullname()))
def __iter__(self):
for key in self._values.keys():
@ -248,10 +258,14 @@ class Table:
def get_table(self):
return self._rows
def _check_columns(self, value: Union[List[Hash], List[dict]]) -> None:
def add_row(self, row: dict):
self._check_columns(row)
self._rows.append(row)
def _check_columns(self, row: dict) -> None:
'''Метод для проверки наличия в хэше только тех полей, которые
соответствуют заданным для таблицы колонкам.'''
for column in value:
for column in row:
if column not in self.columns:
raise VariableError("unknown column value '{}'"
" available: '{}'".format(
@ -359,9 +373,9 @@ class DependenceSource:
except Exception as error:
raise DependenceError('can not calculate using dependencies: {}'
' reason: {}'.format(', '.join(
[subscription.fullname
for subscription
in self._args]),
[subscription.get_fullname()
for subscription
in self._args]),
str(error)))
@property
@ -431,7 +445,8 @@ class DependenceSource:
class VariableNode:
'''Класс ноды соответствующей переменной в дереве переменных.'''
def __init__(self, name, namespace, variable_type=StringType, source=None):
def __init__(self, name, namespace, variable_type=VariableType,
source=None):
self.name = name
if issubclass(variable_type, VariableType):
self.variable_type = variable_type
@ -466,13 +481,13 @@ class VariableNode:
def update_value(self) -> None:
'''Метод для обновления значения переменной с помощью указанного
источника ее значения.'''
print('updating {}'.format(self.fullname))
print('updating {}'.format(self.get_fullname()))
if self.calculating:
raise CyclicVariableError(self.name)
if self._source is None:
raise VariableError("No sources to update variable '{}'".
format(self.fullname))
format(self.get_fullname()))
if isinstance(self._source, DependenceSource):
with self._start_calculate():
@ -531,7 +546,7 @@ class VariableNode:
def _invalidate(self) -> None:
'''Метод для инвалидации данной переменной и всех зависящих от нее
переменных.'''
print('{} is invalidated'.format(self.fullname))
print('{} is invalidated'.format(self.get_fullname()))
if self.value is not None and not self.set_by_user:
self.value = None
for subscriber in self.subscribers:
@ -553,10 +568,9 @@ class VariableNode:
self.update_value()
return self.value
@property
def fullname(self) -> str:
def get_fullname(self) -> str:
if self.namespace:
return "{}.{}".format(self.namespace.fullname, self.name)
return "{}.{}".format(self.namespace.get_fullname(), self.name)
else:
return self.name
@ -566,13 +580,13 @@ class VariableNode:
return self.value[key]
else:
raise VariableError("value '{}' is not found in hash variable"
" '{}'".format(key, self.fullname))
" '{}'".format(key, self.get_fullname()))
else:
raise VariableError("'{}' variable type is not subscriptable.".
format(self.variable_type.name))
def __repr__(self):
return '<Variable: {} with value: {}>'.format(self.fullname,
return '<Variable: {} with value: {}>'.format(self.get_fullname(),
self.value or
'INVALIDATED')
@ -595,10 +609,14 @@ class NamespaceNode:
self.namespaces.update({namespace.name: namespace})
namespace.parent = self
@property
def fullname(self) -> str:
def clear(self):
for namespace in self.namespaces:
namespace.clear()
self.namespaces.clear()
def get_fullname(self) -> str:
if self.parent is not None:
return '{}.{}'.format(self.parent.fullname, self.name)
return '{}.{}'.format(self.parent.get_fullname(), self.name)
else:
return self.name
@ -630,7 +648,7 @@ class NamespaceNode:
return name in self.namespaces or name in self.variables
def __repr__(self):
return '<Namespace: {}>'.format(self.fullname)
return '<Namespace: {}>'.format(self.get_fullname())
class DependenceAPI:
@ -697,7 +715,8 @@ class VariableAPI:
variable.source = source
except DependenceError as error:
raise VariableError('Dependence error: {} in variable: {}'.
format(str(error), variable.fullname))
format(str(error),
variable.get_fullname()))
else:
variable.source = source
@ -712,7 +731,7 @@ class NamespaceAPI:
dependence_fabric: DependenceAPI(),
datavars_root=NamespaceNode('<root>')):
self._datavars = datavars_root
self.namespace_stack = [datavars_root]
self.current_namespace = self._datavars
# Привязываем фабрику переменных.
self._variables_fabric = var_fabric
@ -733,13 +752,12 @@ class NamespaceAPI:
'''Метод для установки корневого пространства имен, которое пока что
будет использоваться для предоставления доступа к переменным.'''
self._datavars = datavars_root
self.namespace_stack = [datavars_root]
self._variables_fabric.current_namespace = self._datavars
def reset(self):
'''Метод для сброса корневого пространства имен.'''
self._datavars = NamespaceNode('<root>')
self.namespace_stack = [self._datavars]
self.current_namespace = self._datavars
self._variables_fabric.current_namespace = self._datavars
self._dependence_fabric.current_namespace = self._datavars
@ -747,16 +765,17 @@ class NamespaceAPI:
@contextmanager
def __call__(self, namespace_name):
'''Метод для создания пространств имен с помощью with.'''
if namespace_name not in self.namespace_stack[-1].namespaces:
if namespace_name not in self.current_namespace.namespaces:
namespace = NamespaceNode(namespace_name,
parent=self.namespace_stack[-1])
parent=self.current_namespace)
else:
namespace = self.namespace_stack[-1].namespaces[namespace_name]
namespace = self.current_namespace.namespaces[namespace_name]
self.current_namespace.add_namespace(namespace)
self.current_namespace = namespace
self.namespace_stack[-1].add_namespace(namespace)
self.namespace_stack.append(namespace)
# Устанавливаем текущее пространство имен фабрике переменных.
self._variables_fabric.current_namespace = namespace
self._variables_fabric.current_namespace = self.current_namespace
# Устанавливаем текущее пространство имен фабрике зависимостей.
self._dependence_fabric.current_namespace = namespace
@ -764,9 +783,9 @@ class NamespaceAPI:
try:
yield self
finally:
current_namespace = self.namespace_stack.pop()
self._variables_fabric.current_namespace = current_namespace
self._dependence_fabric.current_namespace = current_namespace
self.current_namespace = self.current_namespace.parent
self._variables_fabric.current_namespace = self.current_namespace
self._dependence_fabric.current_namespace = self.current_namespace
Dependence = DependenceAPI()

@ -0,0 +1,377 @@
import re
import sys
import os
import importlib
import importlib.util
import site
from calculate.vars.datavars import Variable, Namespace, HashVariable,\
TableVariable, IniCreated, DefaultValue
from calculate.vars.alt_datavars import NamespaceNode, VariableNode,\
StringType, ListType, IntegerType,\
FloatType, IniType
from calculate.utils.gentoo import ProfileWalker
from calculate.utils.fs import readFile
from calculate.utils.files import list_directory
from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
empty, printables, OneOrMore, lineno, line, col, SkipTo,\
LineEnd, Combine
from enum import Enum
class Define(Enum):
assign = 0
append = 1
remove = 2
class CalculateIniParser:
'''Класс парсера calculate.ini файлов.'''
def __init__(self):
self._errors = []
self.operations = {"=": Define.assign,
"+=": Define.append,
"-=": Define.remove}
lbrack = Literal("[")
rbrack = Literal("]")
# comma = Literal(",").suppress()
comment_symbol = Literal(';') | Literal('#')
# Define = self.Define
value_operation = (Literal("=") | Combine(Literal("+") + Literal("="))
| Combine(Literal("-") + Literal("=")))
comment = comment_symbol + Optional(restOfLine)
section_name = Word(printables+'\t', excludeChars='[]')
value_name = Word(printables+'\t', excludeChars='=-+')
# non_comma = Word(printables+'\t', excludeChars=',')
clear_section = lbrack.suppress() + Group(empty) + rbrack.suppress()
section_start = Group(OneOrMore(lbrack.suppress() + section_name
+ rbrack.suppress())
+ (clear_section | ~lbrack())
+ LineEnd().suppress())
# Если содержимое ini-файла не предваряется заголовком секции,
# значит эта строка ошибочна.
unexpected = Group(~section_start + SkipTo(LineEnd(),
include=True))("error")
unexpected.setParseAction(self._unexpected_token)
key_value = (~lbrack + value_name
+ value_operation + empty
+ restOfLine + LineEnd().suppress())
def strip_key_value(tokens):
tokens[0] = tokens[0].strip()
tokens[1] = tokens[1].strip()
key_value.setParseAction(strip_key_value)
self.ini_section_parser = (section_start
+ Group(ZeroOrMore(
Group(key_value | unexpected)))
| unexpected)
self.ini_section_parser.ignore(comment)
def _unexpected_token(self, string, location, tokens):
'''Метод вызываемый парсером, если обнаружена некорректная строка,
предназначен для получения некорректной строки и ее дальнейшего
разбора.'''
error_line = line(location, string).strip()
if error_line:
self._errors.append((error_line, lineno(location, string),
col(location, string)))
@property
def errors(self):
errors = self._errors
self._errors = []
return errors
def parse(self, data):
for tokens, start, end in self.ini_section_parser.scanString(data):
if tokens.getName() == "error":
continue
section, defkeys = tokens
section_list = section.asList()
if section_list[-1] == []:
yield {'clear_section': (section_list[:-1], )}
else:
yield {'start_section': (section.asList(), )}
for defkey in defkeys:
if defkey.getName() == "error":
continue
yield {'define_key': (section.asList(), defkey[0], defkey[2],
self.operations[defkey[1]])}
class NamespaceIniFiller:
'''Класс, предназначенный для наполнения Namespace объекта переменными
из calculate.ini файла.'''
def __init__(self):
self.ini_parser = CalculateIniParser()
def error(self, lineno, error_message):
self.errors.append(lineno, error_message)
def fill(self, namespace, ini_file_text):
'''Метод для разбора calculate.ini файла и добавления всех его
переменных в указанное пространство имен.'''
self.namespace = namespace
self.current_namespace = self.namespace
self.errors = []
for parsed_line in self.ini_parser.parse(ini_file_text):
self._line_processor(**parsed_line)
def _line_processor(self, start_section=None,
clear_section=None,
define_key=None,
error=None, **kwargs):
if start_section is not None:
self.start_section(*start_section)
elif clear_section is not None:
self.clear_section(*clear_section)
elif define_key is not None:
self.define_key(*define_key)
for error in self.ini_parser.errors:
self.set_error(*error)
def start_section(self, sections):
'''Метод для получения доступа и создания пространств имен.'''
self.current_namespace = self.namespace
for section in sections:
if section not in self.current_namespace.namespaces:
self.current_namespace.add_namespace(NamespaceNode(section))
self.current_namespace = self.current_namespace.namespaces[section]
def clear_section(self, sections):
'''Метод для очистки пространства имен.'''
current_namespace = self.namespace
for section in sections:
if section not in current_namespace.namespaces:
return
current_namespace = current_namespace.namespaces[section]
current_namespace.clear()
def change_value(self, key, value):
variable = self.current_namespace[key]
variable.source = value
def define_variable(self, key, value):
'''Метод для создания переменных в calculate.ini файле.'''
print("define variable: '{}' with value: '{}'".format(key, value))
VariableNode(key, self.current_namespace, variable_type=IniType,
source=value)
def append_value(self, key, value):
'''Метод выполняющий действия возложенные на оператор +=.'''
variable = self.current_namespace[key]
variable_value = variable.get_value()
if variable.variable_type is IniType:
value_list = value.split(',')
variable_list = variable_value.split(',')
for item in value_list:
if item not in variable_list:
variable_list.append(item.strip())
variable_value = ','.join(variable_list)
elif variable.variable_type is ListType:
value_list = value.split(',')
for item in value_list:
if item not in variable_value:
variable_value.append(item.strip())
elif variable.variable_type is IntegerType:
variable_value += int(value)
elif variable.variable_type is FloatType:
variable_value += float(value)
variable.source = variable_value
def remove_value(self, key, value):
'''Метод выполняющий действия возложенные на оператор -=.'''
variable = self.current_namespace[key]
variable_value = variable.get_value()
if variable.variable_type is IniType:
value_list = value.split(',')
variable_list = [item.strip() for item in
variable_value.split(',')]
for item in value_list:
if item in variable_list:
variable_list.remove(item.strip())
variable_value = ','.join(variable_list)
elif variable.variable_type is ListType:
value_list = value.split(',')
for item in value_list:
if item in variable_value:
variable_value.remove(item.strip())
elif variable.variable_type is IntegerType:
variable_value -= int(value)
elif variable.variable_type is FloatType:
variable_value -= float(value)
variable.source = variable_value
def define_key(self, section, key, value, optype):
if optype == Define.assign:
if key not in self.current_namespace:
self.define_variable(key, value)
else:
self.change_value(key, value)
elif optype == Define.append:
if key not in self.current_namespace:
self.define_variable(key, value)
else:
self.append_value(key, value)
elif optype == Define.remove:
if key not in self.current_namespace:
self.define_variable(key, value)
else:
self.remove_value(key, value)
def set_error(self, line, lineno, col):
self.error(lineno, "Syntax error: {}".format(line))
class NamespaceIniFillerStrict(NamespaceIniFiller):
"""Объект используемый для наполнения Namespace объекта переменными
из calculate.ini файла, с возможность определять только
"""
availableSection = ["custom"]
def fill(self, namespace, data):
self.canCreate = False
for newns in self.availableSection:
if newns not in namespace:
namespace.add_namespace(Namespace(newns))
super().fill(namespace, data)
def start_section(self, sections):
self.current_namespace = self.namespace
self.canCreate = sections[0] in self.availableSection
for section in sections:
if section not in self.current_namespace.childs:
if isinstance(self.current_namespace,
TableVariable) or self.canCreate:
self.current_namespace.add_namespace(Namespace(section))
else:
self.current_namespace = None
self.current_namespace = self.current_namespace[section]
# def clear_section(self, sections):
# current_namespace = self.namespace
# self.canCreate = sections[0] in self.availableSection
# for section in sections:
# if section not in current_namespace.childs:
# return
# current_namespace = current_namespace[section]
# current_namespace.clear_childs()
def define_variable(self, key, value):
if not self.canCreate:
pass
var = Variable(key)
var.add_property(IniCreated(value))
self.current_namespace.add_variable(var)
def change_value(self, key, value):
var = self.current_namespace[key]
var.set_value(value)
if isinstance(var, HashVariable.HashValue):
var = var.master_variable
value = var.get_value()
prop = var.find_property(DefaultValue)
if prop:
prop.value = value
else:
var.add_property(DefaultValue(value))
var.invalidate()
class VariableLoader:
'''Класс, используемый для загрузки переменных из python модуля.'''
re_upper = re.compile("(.)([A-Z])")
def _get_varlike_attrs(self, obj):
'''Метод для получения списка аттрибутов похожих на переменные.'''
for attrname in (x for x in dir(obj) if x[:1].isupper()):
yield self.re_upper.sub(r"\1_\2", attrname).lower(), \
getattr(obj, attrname)
def fill(self, namespace, dirpath, package):
'''Загрузить в namespace переменные из указанных модулей.'''
for fullfn in list_directory(dirpath, fullpath=True):
dn, fn = os.path.split(fullfn)
if os.path.isdir(fullfn):
newns = namespace.add_namespace(Namespace(fn))
self.fill(newns, fullfn, "{}.{}".format(package, fn))
elif fn.endswith(".py"):
module = self._load_module_source(package, fn, fullfn)
for varname, cls in self._get_varlike_attrs(module):
if Variable.is_implementation(cls):
namespace.add_variable(cls(varname))
elif HashVariable.is_implementation(cls) or \
TableVariable.is_implementation(cls) or \
Namespace.is_implementation(cls):
_newns = namespace.add_namespace(cls(varname,
namespace))
for _varname, _cls in self._get_varlike_attrs(cls):
if Variable.is_implementation(_cls):
_newns.add_variable(_cls(_varname))
@classmethod
def default(cls):
site_dirs = [os.path.normpath(x) for x in site.getsitepackages()]
for site_dir in site_dirs:
calculate_dir = os.path.join(site_dir, "calculate/vars")
if os.path.exists(calculate_dir):
return (calculate_dir, "calculate.vars")
def _load_module_source(self, package, name, path):
if name.startswith('calculate.vars.'):
full_name = name
else:
full_name = '.'.join([package, name])
if full_name in sys.modules:
return sys.modules[full_name]
spec = importlib.util.spec_from_file_location(full_name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
sys.modules[full_name] = module
return module
class ProfileFiller:
"""
Заполнитель значений переменных из файлов calculate.ini в профилях
"""
basename = "calculate.ini"
def get_repository_map(self, namespace):
return {
x.name.get_value(): x.path.get_value()
for x in namespace.os.gentoo.repositories
}
def fill(self, namespace, profile_path):
nif = NamespaceIniFillerStrict()
pw = ProfileWalker(self.basename, self.get_repository_map(namespace))
for fn in pw.find(profile_path):
nif.fill(namespace, readFile(fn))

@ -4,6 +4,7 @@ from calculate.vars.alt_datavars import NamespaceNode, VariableNode,\
CyclicVariableError, HashType,\
StringType, IntegerType, VariableType,\
VariableError, TableType
from calculate.vars.alt_vars_loader import NamespaceIniFiller
from calculate.utils.files import stderr_devnull
@ -16,7 +17,7 @@ class TestNamespace:
assert namespace_1.namespaces == dict()
assert namespace_1.variables == dict()
assert namespace_1.fullname == 'namespace_1'
assert namespace_1.get_fullname() == 'namespace_1'
def test_if_namespace_node_added_in_an_other_namespace_node__a_parent_namespace_node_contains_child_namespace_in_its_namespaces_dictionary_and_child_namespace_object_has_parent_namespace_in_its_parent_attribute_and_its_fullname_method_returns_names_of_both_namespaces(self):
namespace_1 = NamespaceNode('namespace_1')
@ -25,7 +26,7 @@ class TestNamespace:
assert namespace_1.namespaces == {'namespace_2': namespace_2}
assert namespace_1.variables == dict()
assert namespace_1.fullname == 'namespace_1'
assert namespace_1.get_fullname() == 'namespace_1'
assert namespace_1.namespace_2 == namespace_2
assert namespace_1['namespace_2'] == namespace_2
@ -33,7 +34,7 @@ class TestNamespace:
assert namespace_2.namespaces == dict()
assert namespace_2.variables == dict()
assert namespace_2.parent == namespace_1
assert namespace_2.fullname == 'namespace_1.namespace_2'
assert namespace_2.get_fullname() == 'namespace_1.namespace_2'
def test_if_two_VariableNode_objects_are_initialized_and_added_to_a_namespace__the_NamespaceNode_object_contains_this_variables_and_can_be_used_to_get_variables_values_and_variables_have_namespace_name_in_their_fullnames(self):
namespace_1 = NamespaceNode('namespace_1')
@ -43,7 +44,7 @@ class TestNamespace:
assert namespace_1.namespaces == dict()
assert namespace_1.variables == {'var_1': variable_1,
'var_2': variable_2}
assert namespace_1.fullname == 'namespace_1'
assert namespace_1.get_fullname() == 'namespace_1'
assert namespace_1.var_1 == 'value_1'
assert namespace_1['var_1'] == variable_1
@ -51,8 +52,8 @@ class TestNamespace:
assert namespace_1.var_2 == 'value_2'
assert namespace_1['var_2'] == variable_2
assert namespace_1['var_1'].fullname == 'namespace_1.var_1'
assert namespace_1['var_2'].fullname == 'namespace_1.var_2'
assert namespace_1['var_1'].get_fullname() == 'namespace_1.var_1'
assert namespace_1['var_2'].get_fullname() == 'namespace_1.var_2'
def test_compare_two_dependencies_equal(self):
namespace_1 = NamespaceNode('namespace_1')
@ -439,7 +440,7 @@ class TestNamespace:
if row['name'] == 'name_2':
return row['value']
Variable('var_2', type=StringType,
source=Dependence('os.var_1',
source=Dependence('namespace_1.var_1',
depend=depend_function))
assert datavars.namespace_1.var_2 == 'value_2'
@ -458,7 +459,8 @@ class TestNamespace:
with Namespace('os'):
def source_function(chroot_path):
if chroot_path.value == '/':
return config()
with stderr_devnull():
return config()
Variable('config', source=Dependence('main.chroot',
depend=source_function),
type=VariableType)
@ -471,3 +473,95 @@ class TestNamespace:
source=Dependence('.config', depend=config_source))
assert datavars.os.repositories.get_table() == value
# Тестируем теперь заполнитель переменных из calculate.ini
def test_if_calculate_ini_file_is_used_just_to_create_some_variables__the_namespace_filler_will_create_them(self):
Namespace.reset()
datavars = Namespace.datavars
calculate_ini_text = """
[os]
test = 123
[os][linux]
shortname = CLD
fullname = Calculate Linux Desktop
"""
namespace_filler = NamespaceIniFiller()
namespace_filler.fill(datavars, calculate_ini_text)
assert datavars.os.test == '123'
assert datavars.os.linux.shortname == 'CLD'
assert datavars.os.linux.fullname == 'Calculate Linux Desktop'
def test_if_a_calculate_ini_file_is_used_to_create_some_variables_and_an_other_one_is_used_to_update_value_of_the_variable__the_namespace_filler_creates_them_using_a_first_calculate_ini_and_then_changes_value_using_a_second_calculate_ini(self):
Namespace.reset()
datavars = Namespace.datavars
namespace_filler = NamespaceIniFiller()
first_ini_text = """
[os]
test = 123
[os][linux]
shortname = CLD
fullname = Calculate Linux Desktop
"""
namespace_filler.fill(datavars, first_ini_text)
assert datavars.os.test == '123'
assert datavars.os.linux.shortname == 'CLD'
assert datavars.os.linux.fullname == 'Calculate Linux Desktop'
second_ini_text = """
[os][linux]
shortname = CLDX
"""
namespace_filler.fill(datavars, second_ini_text)
assert datavars.os.linux.shortname == 'CLDX'
def test_if_calculate_ini_file_creates_a_variable_and_then_uses_append_operation_to_append_any_value_to_the_created_variable__the_namespace_filler_appends_the_value_to_the_created_variable(self):
Namespace.reset()
datavars = Namespace.datavars
namespace_filler = NamespaceIniFiller()
first_ini_text = """
[os]
test = 123
[os]
test += 345
"""
namespace_filler.fill(datavars, first_ini_text)
assert datavars.os.test == "123,345"
second_ini_text = """
[os]
test += asdf,qwer,zxcv
"""
namespace_filler.fill(datavars, second_ini_text)
assert datavars.os.test == "123,345,asdf,qwer,zxcv"
def test_if_calculate_ini_file_creates_a_variable_and_then_uses_remove_operation_to_remove_any_values_in_the_created_variable__the_namespace_filler_removes_the_values_from_the_created_variable(self):
Namespace.reset()
datavars = Namespace.datavars
namespace_filler = NamespaceIniFiller()
first_ini_text = """
[os]
test = 123,345,asdf,qwer,zxcv
"""
namespace_filler.fill(datavars, first_ini_text)
assert datavars.os.test == "123,345,asdf,qwer,zxcv"
second_ini_text = """
[os]
test -= 345
"""
namespace_filler.fill(datavars, second_ini_text)
assert datavars.os.test == "123,asdf,qwer,zxcv"
third_ini_text = """
[os]
test -= asdf,zxcv
"""
namespace_filler.fill(datavars, third_ini_text)
assert datavars.os.test == "123,qwer"

Loading…
Cancel
Save