From 632b88b79fc94d5d6d4df60f16b0122ec7e18a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=D0=BE=D0=B2=20=D0=94=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=81?= Date: Fri, 7 Aug 2020 18:59:41 +0300 Subject: [PATCH] The datavars loader is done. The ini datavars loader is basically implemented. --- calculate/utils/files.py | 4 +- calculate/vars/alt_datavars.py | 58 ++++++++-- calculate/vars/alt_vars_loader.py | 106 +++++++++++++----- tests/vars/test_alt_vars.py | 31 +++-- .../testfiles/variables_1/level/__init__.py | 62 ++++++++++ .../variables_1/level/level_2/__init__.py | 7 ++ .../testfiles/variables_1/main/__init__.py | 5 + .../vars/testfiles/variables_1/os/__init__.py | 36 ++++++ 8 files changed, 259 insertions(+), 50 deletions(-) create mode 100644 tests/vars/testfiles/variables_1/level/__init__.py create mode 100644 tests/vars/testfiles/variables_1/level/level_2/__init__.py create mode 100644 tests/vars/testfiles/variables_1/main/__init__.py create mode 100644 tests/vars/testfiles/variables_1/os/__init__.py diff --git a/calculate/utils/files.py b/calculate/utils/files.py index ff613f9..81bf9e7 100644 --- a/calculate/utils/files.py +++ b/calculate/utils/files.py @@ -351,8 +351,8 @@ def read_file(file_path): return opened_file.read() except (OSError, IOError) as error: mod, lineno = get_traceback_caller(*sys.exc_info()) - FilesError("file read error, {0}({1}:{2})". - format(str(error), mod, lineno)) + raise FilesError("file read error, {0}({1}:{2})". + format(str(error), mod, lineno)) def write_file(file_path): diff --git a/calculate/vars/alt_datavars.py b/calculate/vars/alt_datavars.py index 0967d72..ffecd09 100644 --- a/calculate/vars/alt_datavars.py +++ b/calculate/vars/alt_datavars.py @@ -349,11 +349,12 @@ class VariableWrapper: class DependenceSource: '''Класс зависимости как источника значения переменной.''' - def __init__(self, variables, depend=None): + def __init__(self, variables: tuple, depend=None): self.error = None self._args = variables self.depend_function = depend self._subscriptions = set() + self._args_founded = False def check(self) -> None: '''Метод для запуска проверки корректности функции зависимости, а также @@ -388,6 +389,19 @@ class DependenceSource: in self._args]), str(error))) + def get_args(self, namespace): + if not self._args_founded: + for index in range(0, len(self._args)): + if isinstance(self._args[index], str): + variable = Dependence._find_variable( + self._args[index], + current_namespace=namespace) + if variable is None: + raise DependenceError("variable '{}' not found". + format(variable)) + self._args[index] = variable + self._args_founded = True + @property def subscriptions(self) -> set: return self._subscriptions @@ -500,6 +514,8 @@ class VariableNode: format(self.get_fullname())) if isinstance(self._source, DependenceSource): + print('NAMESPACE: {}'.format(self.namespace)) + self._source.get_args(self.namespace) with self._start_calculate(): try: value = self._source.calculate_value() @@ -514,6 +530,9 @@ class VariableNode: self._subscriptions = self._source.subscriptions except CyclicVariableError as error: raise CyclicVariableError(self.name, *error.queue) + except DependenceError as error: + raise VariableError('{}: {}'.format(self.get_fullname(), + str(error))) else: value = self._source @@ -556,7 +575,7 @@ class VariableNode: def _invalidate(self) -> None: '''Метод для инвалидации данной переменной и всех зависящих от нее переменных.''' - print('{} is invalidated'.format(self.get_fullname())) + # print('{} is invalidated'.format(self.get_fullname())) if self.value is not None and not self.set_by_user: self.value = None for subscriber in self.subscribers: @@ -639,7 +658,10 @@ class NamespaceNode: if name in self.namespaces: return self.namespaces[name] elif name in self.variables: - return self.variables[name].get_value() + variable = self.variables[name] + if variable.variable_type is TableType: + return variable.get_value().get_table() + return variable.get_value() else: raise VariableError("'{variable_name}' is not found in the" " namespace '{namespace_name}'".format( @@ -674,23 +696,33 @@ class DependenceAPI: def __call__(self, *variables, depend=None): subscriptions = list() for variable in variables: - if isinstance(variable, str): - variable = self._find_variable(variable) - if variable is None: - raise DependenceError("variable '{}' not found".format( - variable)) - elif not isinstance(variable, VariableNode): + # Поиск переменных теперь происходит при обновлении значения + # переменной. + + # if isinstance(variable, str): + # variable = self._find_variable(variable) + # if variable is None: + # raise DependenceError("variable '{}' not found".format( + # variable)) + # elif not isinstance(variable, VariableNode): + # raise DependenceError("dependence variables must be 'str' or" + # " 'VariableNode' not '{}'".format( + # type(variable))) + if not (isinstance(variable, str) or + isinstance(variable, VariableNode)): raise DependenceError("dependence variables must be 'str' or" " 'VariableNode' not '{}'".format( type(variable))) subscriptions.append(variable) return DependenceSource(subscriptions, depend=depend) - def _find_variable(self, variable_name): + def _find_variable(self, variable_name, current_namespace=None): '''Метод для поиска переменной в пространствах имен.''' + if current_namespace is None: + current_namespace = self.current_namespace name_parts = variable_name.split('.') if not name_parts[0]: - namespace = self.current_namespace + namespace = current_namespace for index in range(1, len(name_parts)): if not name_parts[index]: namespace = namespace.parent @@ -718,9 +750,13 @@ class VariableAPI: readonly=False, force=False): '''Метод для создания переменных внутри with Namespace('name').''' if name not in self.current_namespace.variables: + print('CREATE VARIABLE: {}'.format('{}.{}'.format( + self.current_namespace.get_fullname(), + name))) variable = VariableNode(name, self.current_namespace, variable_type=type) else: + print('MODIFY VARIABLE: {}'.format(name)) variable = self.current_namespace[name] if isinstance(source, DependenceSource): diff --git a/calculate/vars/alt_vars_loader.py b/calculate/vars/alt_vars_loader.py index ca175b0..7bfb747 100644 --- a/calculate/vars/alt_vars_loader.py +++ b/calculate/vars/alt_vars_loader.py @@ -4,19 +4,18 @@ import os import importlib import importlib.util import site -from calculate.vars.datavars import Variable, Namespace, HashVariable,\ - TableVariable, IniCreated, DefaultValue +from calculate.vars.datavars import Variable, HashVariable, TableVariable from calculate.vars.alt_datavars import NamespaceNode, VariableNode,\ ListType, IntegerType,\ FloatType, IniType, TableType,\ Namespace from calculate.utils.gentoo import ProfileWalker -from calculate.utils.fs import readFile -from calculate.utils.files import list_directory +from calculate.utils.files import list_directory, read_file, FilesError from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\ empty, printables, OneOrMore, lineno, line, col, SkipTo,\ LineEnd, Combine, nums from enum import Enum +from contextlib import contextmanager class Define(Enum): @@ -346,25 +345,69 @@ class NamespaceIniFiller: class VariableLoader: + basename = "calculate.ini" + def load_variables(self, directory_path) -> NamespaceNode: root = Namespace.datavars package = '.'.join(directory_path.split('/')) - self._fill(root, directory_path, package) + self._fill_from_package(root, directory_path, package) return root - def _fill(self, current_namespace: NamespaceNode, - directory_path: str, package: str) -> None: - for file_name in list_directory(directory_path): - file_path = os.path.join(directory_path, file_name) - if os.path.isdir(file_path): - namespace = NamespaceNode(file_name) - current_namespace.add_namespace(namespace) - self._fill(namespace, file_path, "{}.{}".format(package, - file_name)) - elif file_name.endswith('.py'): - file_name = file_name[:-3] - Namespace.set_current_namespace(current_namespace) - importlib.import_module('{}.{}'.format(package, file_name)) + def _fill_from_package(self, current_namespace: NamespaceNode, + directory_path: str, package: str) -> None: + print('FILL -> {}'.format(current_namespace)) + file_nodes = [] + directory_nodes = [] + # Просматриваем директорию + for node in os.scandir(directory_path): + if node.is_dir(): + directory_nodes.append(node) + elif node.is_file() and node.name.endswith('.py'): + file_nodes.append(node) + + # Сначала загружаем переменные из файлов. + for file_node in file_nodes: + file_name = file_node.name[:-3] + Namespace.set_current_namespace(current_namespace) + with self.test(file_name, current_namespace): + # importlib.import_module('{}.{}'.format(package, file_name)) + spec = importlib.util.spec_from_file_location( + '{}.{}'.format(package, file_name), + file_node.path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Обходим остальные директории. + for directory_node in directory_nodes: + namespace = NamespaceNode(directory_node.name) + current_namespace.add_namespace(namespace) + self._fill_from_package(namespace, directory_node.path, + '{}.{}'.format(package, + directory_node.name)) + + def _fill_from_ini(self, datavars, profile_path): + ini_filler = NamespaceIniFiller() + profile_walker = ProfileWalker(self.basename, + self.get_repository_map(datavars)) + for file_path in profile_walker.find(profile_path): + try: + ini_file_text = read_file(file_path) + ini_filler.fill(datavars, ini_file_text) + except FilesError: + pass + + def get_repository_map(self, datavars): + return {repo['name']: repo['path'] + for repo in datavars.os.gentoo.repositories} + + @contextmanager + def test(self, file_name, namespace): + print('IMPORT: {}.{}'.format(namespace.get_fullname(), file_name)) + try: + yield self + finally: + print('IMPORTED FROM: {}.{}'.format(namespace.get_fullname(), + file_name)) class OldVariableLoader: @@ -422,20 +465,23 @@ class OldVariableLoader: return module -class ProfileFiller: +class OldProfileFiller: """ Заполнитель значений переменных из файлов calculate.ini в профилях """ basename = "calculate.ini" - def get_repository_map(self, namespace): - return { - x.name.get_value(): x.path.get_value() - for x in namespace.os.gentoo.repositories - } - - def fill(self, namespace, profile_path): - nif = NamespaceIniFillerStrict() - pw = ProfileWalker(self.basename, self.get_repository_map(namespace)) - for fn in pw.find(profile_path): - nif.fill(namespace, readFile(fn)) + def get_repository_map(self, datavars): + return {repo['name']: repo['path'] + for repo in datavars.os.gentoo.repositories} + + def fill(self, datavars, profile_path): + ini_filler = NamespaceIniFiller() + profile_walker = ProfileWalker(self.basename, + self.get_repository_map(datavars)) + for file_path in profile_walker.find(profile_path): + try: + ini_file_text = read_file(file_path) + ini_filler.fill(datavars, ini_file_text) + except FilesError: + pass diff --git a/tests/vars/test_alt_vars.py b/tests/vars/test_alt_vars.py index bacd388..641b1c2 100644 --- a/tests/vars/test_alt_vars.py +++ b/tests/vars/test_alt_vars.py @@ -586,7 +586,7 @@ class TestNamespace: Variable('repositories', type=TableType, source=Dependence('.config', depend=config_source)) - assert datavars.os.repositories.get_table() == value + assert datavars.os.repositories == value # Тестируем теперь заполнитель переменных из calculate.ini def test_if_calculate_ini_file_is_used_just_to_create_some_variables__the_namespace_filler_will_create_them(self): @@ -763,7 +763,7 @@ value = value_2 """ namespace_filler.fill(datavars, first_ini_text) - assert datavars.custom.test.get_table() ==\ + assert datavars.custom.test ==\ [{'name': 'name_0', 'value': 'value_0'}, {'name': 'name_1', 'value': 'value_1'}, {'name': 'name_2', 'value': 'value_2'}] @@ -835,9 +835,9 @@ value = weird_value """ namespace_filler.fill(datavars, first_ini_text) - assert datavars.custom.test.get_table() ==\ + assert datavars.custom.test ==\ [{'name': 'name_0', 'value': 'value_0'}] - assert datavars.os.test.get_table() ==\ + assert datavars.os.test ==\ [{'name': 'strange_name', 'value': 'weird_value'}] def test_if_calculate_ini_file_is_used_for_modifying_of_the_table_from_calculate_ini_file__the_NamespaceIniFiller_object_modifies_the_table(self): @@ -857,7 +857,7 @@ value = value_1 """ namespace_filler.fill(datavars, first_ini_text) - assert datavars.custom.test.get_table() ==\ + assert datavars.custom.test ==\ [{'name': 'name_0', 'value': 'value_0'}, {'name': 'name_1', 'value': 'value_1'}] @@ -868,7 +868,7 @@ value = another_value """ namespace_filler.fill(datavars, second_ini_text) - assert datavars.custom.test.get_table() ==\ + assert datavars.custom.test ==\ [{'name': 'name_0', 'value': 'value_0'}, {'name': 'other_name', 'value': 'another_value'}] @@ -896,7 +896,7 @@ value = another_value """ namespace_filler.fill(datavars, first_ini_text) - assert datavars.namespace_1.test.get_table() ==\ + assert datavars.namespace_1.test ==\ [{'name': 'new_name', 'value': 'other_value'}, {'name': 'common_name', 'value': 'another_value'}, {'name': 'name_3', 'value': 'value_3'}] @@ -910,3 +910,20 @@ value = another_value assert datavars.main.strange_variable == 'weird_value' assert datavars.main.plain_variable is True + + def test_loader_2(self): + Namespace.reset() + loader = VariableLoader() + datavars = loader.load_variables('tests/vars/testfiles/variables_1') + + assert datavars.main.chroot == '/' + assert datavars.level.simple == "simple value" + assert datavars.level.use_local_simple == "Using simple value" + assert datavars.level.use_full_simple == "Using simple value" + assert datavars.level.device_child == "hdd" + assert datavars.level.device == [{"dev": "/dev/sda", + "type": "hdd", + "name": "Samsung SSD"}, + {"dev": "/dev/sdb", + "type": "flash", + "name": "Transcend 64GB"}] diff --git a/tests/vars/testfiles/variables_1/level/__init__.py b/tests/vars/testfiles/variables_1/level/__init__.py new file mode 100644 index 0000000..cac0562 --- /dev/null +++ b/tests/vars/testfiles/variables_1/level/__init__.py @@ -0,0 +1,62 @@ +from calculate.vars.alt_datavars import Namespace, Variable, Dependence,\ + StringType, HashType, TableType,\ + ListType, FloatType + + +Variable('simple', type=StringType, source='simple value') + +Variable('use_local_simple', type=StringType, + source=Dependence('.simple', + depend=lambda simple: 'Using {}'.format( + simple.value))) + +Variable('use_full_simple', type=StringType, + source=Dependence('level.simple', + depend=lambda simple: 'Using {}'.format( + simple.value))) + +Variable('disks', type=ListType, + source=["/dev/sda1", "/dev/sda2", "/dev/sda3"]) + +Variable('version', type=FloatType, source='1.0') + +Variable('my_shortname', type=StringType, source='CLD') + +Variable('linux', type=HashType, + source=Dependence('.version', '.my_shortname', + depend=lambda version, my_shortname: + {'version': version.value, + 'shortname': my_shortname.value})) + +Variable('shortname_test', type=StringType, + source=Dependence('.linux.shortname', + depend=lambda shortname: '{} test'.format( + shortname.value))) + +Variable('device_list', type=ListType, + source=["/dev/sda", "/dev/sdb"]) + + +def get_device_table(device_list): + map_data = {'/dev/sda': ["hdd", "Samsung SSD"], + '/dev/sdb': ["flash", "Transcend 64GB"], + '/dev/sdc': ["usbhdd", "WD 1TB"]} + default_value = ["hdd", "Unknown"] + print('device_list = {}'.format(device_list.value)) + return [{"dev": device, + "type": map_data.get(device, default_value)[0], + "name": map_data.get(device, default_value)[1]} + for device in device_list.value] + + +Variable('device', type=TableType, source=Dependence('.device_list', + depend=get_device_table)) + +Variable('device_child', type=StringType, + source=Dependence('.device', + depend=lambda device: device.value[0]['type'])) + +with Namespace('level_3'): + Variable('my_var_1', type=StringType, source='testing') + + Variable('my_var_2', type=StringType, source='testing_2') diff --git a/tests/vars/testfiles/variables_1/level/level_2/__init__.py b/tests/vars/testfiles/variables_1/level/level_2/__init__.py new file mode 100644 index 0000000..e0854fc --- /dev/null +++ b/tests/vars/testfiles/variables_1/level/level_2/__init__.py @@ -0,0 +1,7 @@ +from calculate.vars.alt_datavars import Variable, StringType, Dependence + + +Variable('vargetter', type=StringType, + source=Dependence('main.chroot', + depend=lambda chroot: + '{} test'.format(chroot.value))) diff --git a/tests/vars/testfiles/variables_1/main/__init__.py b/tests/vars/testfiles/variables_1/main/__init__.py new file mode 100644 index 0000000..e460481 --- /dev/null +++ b/tests/vars/testfiles/variables_1/main/__init__.py @@ -0,0 +1,5 @@ +from calculate.vars.alt_datavars import Variable, StringType + + +Variable('chroot', type=StringType, source='/', readonly=True) +print('chroot really created') diff --git a/tests/vars/testfiles/variables_1/os/__init__.py b/tests/vars/testfiles/variables_1/os/__init__.py new file mode 100644 index 0000000..36526e5 --- /dev/null +++ b/tests/vars/testfiles/variables_1/os/__init__.py @@ -0,0 +1,36 @@ +from calculate.vars.alt_datavars import Namespace, Variable, Dependence,\ + StringType, HashType, TableType + +with Namespace('linux'): + Variable('shortname', source='', type=StringType) + + Variable('ver', source='', type=StringType) + + Variable('fullname', source='', type=StringType) + + Variable('subname', source='', type=StringType) + + Variable('arch', source='', type=StringType) + + Variable('test', source='', type=StringType) + + def get_title(subname, fullname, ver): + if subname.value: + return '{} {} {}'.format(fullname.value, subname.value, ver.value) + else: + return '{} {}'.format(fullname.value, ver.value) + Variable('title', type=StringType, + source=Dependence('.subname', '.fullname', '.ver', + depend=get_title)) + +Variable('hashvar', source={'value1': 'test1', + 'value2': 'test2'}, type=HashType) + +Variable('calculate', type=StringType, + source=lambda hashvar: "{} {}".format(hashvar.value['value1'], + hashvar.value['value2'])) + +Variable('tablevar', type=TableType, source=[{"dev": "/dev/sdb1", + "mount": "/"}, + {"dev": "/dev/sdb2", + "mount": "/var/calculate"}])