The datavars loader is done. The ini datavars loader is basically implemented.

calculate/utils/files.py

@@ -351,8 +351,8 @@ def read_file(file_path):
                 return opened_file.read()
     except (OSError, IOError) as error:
         mod, lineno = get_traceback_caller(*sys.exc_info())
-        FilesError("file read error, {0}({1}:{2})".
+        raise FilesError("file read error, {0}({1}:{2})".
-                   format(str(error), mod, lineno))
+                         format(str(error), mod, lineno))
 
 
 def write_file(file_path):

calculate/vars/alt_datavars.py

@@ -349,11 +349,12 @@ class VariableWrapper:
 
 class DependenceSource:
     '''Класс зависимости как источника значения переменной.'''
-    def __init__(self, variables, depend=None):
+    def __init__(self, variables: tuple, depend=None):
         self.error = None
         self._args = variables
         self.depend_function = depend
         self._subscriptions = set()
+        self._args_founded = False
 
     def check(self) -> None:
         '''Метод для запуска проверки корректности функции зависимости, а также
@@ -388,6 +389,19 @@ class DependenceSource:
                                                     in self._args]),
                                                        str(error)))
 
+    def get_args(self, namespace):
+        if not self._args_founded:
+            for index in range(0, len(self._args)):
+                if isinstance(self._args[index], str):
+                    variable = Dependence._find_variable(
+                                                   self._args[index],
+                                                   current_namespace=namespace)
+                    if variable is None:
+                        raise DependenceError("variable '{}' not found".
+                                              format(variable))
+                    self._args[index] = variable
+            self._args_founded = True
+
     @property
     def subscriptions(self) -> set:
         return self._subscriptions
@@ -500,6 +514,8 @@ class VariableNode:
                                 format(self.get_fullname()))
 
         if isinstance(self._source, DependenceSource):
+            print('NAMESPACE: {}'.format(self.namespace))
+            self._source.get_args(self.namespace)
             with self._start_calculate():
                 try:
                     value = self._source.calculate_value()
@@ -514,6 +530,9 @@ class VariableNode:
                     self._subscriptions = self._source.subscriptions
                 except CyclicVariableError as error:
                     raise CyclicVariableError(self.name, *error.queue)
+                except DependenceError as error:
+                    raise VariableError('{}: {}'.format(self.get_fullname(),
+                                                        str(error)))
         else:
             value = self._source
 
@@ -556,7 +575,7 @@ class VariableNode:
     def _invalidate(self) -> None:
         '''Метод для инвалидации данной переменной и всех зависящих от нее
         переменных.'''
-        print('{} is invalidated'.format(self.get_fullname()))
+        # print('{} is invalidated'.format(self.get_fullname()))
         if self.value is not None and not self.set_by_user:
             self.value = None
             for subscriber in self.subscribers:
@@ -639,7 +658,10 @@ class NamespaceNode:
         if name in self.namespaces:
             return self.namespaces[name]
         elif name in self.variables:
-            return self.variables[name].get_value()
+            variable = self.variables[name]
+            if variable.variable_type is TableType:
+                return variable.get_value().get_table()
+            return variable.get_value()
         else:
             raise VariableError("'{variable_name}' is not found in the"
                                 " namespace '{namespace_name}'".format(
@@ -674,23 +696,33 @@ class DependenceAPI:
     def __call__(self, *variables, depend=None):
         subscriptions = list()
         for variable in variables:
-            if isinstance(variable, str):
+            # Поиск переменных теперь происходит при обновлении значения
-                variable = self._find_variable(variable)
+            # переменной.
-                if variable is None:
+
-                    raise DependenceError("variable '{}' not found".format(
+            # if isinstance(variable, str):
-                                                                     variable))
+            #     variable = self._find_variable(variable)
-            elif not isinstance(variable, VariableNode):
+            #     if variable is None:
+            #         raise DependenceError("variable '{}' not found".format(
+            #                                                        variable))
+            # elif not isinstance(variable, VariableNode):
+            #     raise DependenceError("dependence variables must be 'str' or"
+            #                           " 'VariableNode' not '{}'".format(
+            #                                                  type(variable)))
+            if not (isinstance(variable, str) or
+                    isinstance(variable, VariableNode)):
                 raise DependenceError("dependence variables must be 'str' or"
                                       " 'VariableNode' not '{}'".format(
                                                                type(variable)))
             subscriptions.append(variable)
         return DependenceSource(subscriptions, depend=depend)
 
-    def _find_variable(self, variable_name):
+    def _find_variable(self, variable_name, current_namespace=None):
         '''Метод для поиска переменной в пространствах имен.'''
+        if current_namespace is None:
+            current_namespace = self.current_namespace
         name_parts = variable_name.split('.')
         if not name_parts[0]:
-            namespace = self.current_namespace
+            namespace = current_namespace
             for index in range(1, len(name_parts)):
                 if not name_parts[index]:
                     namespace = namespace.parent
@@ -718,9 +750,13 @@ class VariableAPI:
                  readonly=False, force=False):
         '''Метод для создания переменных внутри with Namespace('name').'''
         if name not in self.current_namespace.variables:
+            print('CREATE VARIABLE: {}'.format('{}.{}'.format(
+                                         self.current_namespace.get_fullname(),
+                                         name)))
             variable = VariableNode(name, self.current_namespace,
                                     variable_type=type)
         else:
+            print('MODIFY VARIABLE: {}'.format(name))
             variable = self.current_namespace[name]
 
         if isinstance(source, DependenceSource):

calculate/vars/alt_vars_loader.py

@@ -4,19 +4,18 @@ import os
 import importlib
 import importlib.util
 import site
-from calculate.vars.datavars import Variable, Namespace, HashVariable,\
+from calculate.vars.datavars import Variable, HashVariable, TableVariable
-                                    TableVariable, IniCreated, DefaultValue
 from calculate.vars.alt_datavars import NamespaceNode, VariableNode,\
                                         ListType, IntegerType,\
                                         FloatType, IniType, TableType,\
                                         Namespace
 from calculate.utils.gentoo import ProfileWalker
-from calculate.utils.fs import readFile
+from calculate.utils.files import list_directory, read_file, FilesError
-from calculate.utils.files import list_directory
 from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
                       empty, printables, OneOrMore, lineno, line, col, SkipTo,\
                       LineEnd, Combine, nums
 from enum import Enum
+from contextlib import contextmanager
 
 
 class Define(Enum):
@@ -346,25 +345,69 @@ class NamespaceIniFiller:
 
 
 class VariableLoader:
+    basename = "calculate.ini"
+
     def load_variables(self, directory_path) -> NamespaceNode:
         root = Namespace.datavars
         package = '.'.join(directory_path.split('/'))
-        self._fill(root, directory_path, package)
+        self._fill_from_package(root, directory_path, package)
         return root
 
-    def _fill(self, current_namespace: NamespaceNode,
+    def _fill_from_package(self, current_namespace: NamespaceNode,
-              directory_path: str, package: str) -> None:
+                           directory_path: str, package: str) -> None:
-        for file_name in list_directory(directory_path):
+        print('FILL -> {}'.format(current_namespace))
-            file_path = os.path.join(directory_path, file_name)
+        file_nodes = []
-            if os.path.isdir(file_path):
+        directory_nodes = []
-                namespace = NamespaceNode(file_name)
+        # Просматриваем директорию
-                current_namespace.add_namespace(namespace)
+        for node in os.scandir(directory_path):
-                self._fill(namespace, file_path, "{}.{}".format(package,
+            if node.is_dir():
-                                                                file_name))
+                directory_nodes.append(node)
-            elif file_name.endswith('.py'):
+            elif node.is_file() and node.name.endswith('.py'):
-                file_name = file_name[:-3]
+                file_nodes.append(node)
-                Namespace.set_current_namespace(current_namespace)
+
-                importlib.import_module('{}.{}'.format(package, file_name))
+        # Сначала загружаем переменные из файлов.
+        for file_node in file_nodes:
+            file_name = file_node.name[:-3]
+            Namespace.set_current_namespace(current_namespace)
+            with self.test(file_name, current_namespace):
+                # importlib.import_module('{}.{}'.format(package, file_name))
+                spec = importlib.util.spec_from_file_location(
+                                            '{}.{}'.format(package, file_name),
+                                            file_node.path)
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+
+        # Обходим остальные директории.
+        for directory_node in directory_nodes:
+            namespace = NamespaceNode(directory_node.name)
+            current_namespace.add_namespace(namespace)
+            self._fill_from_package(namespace, directory_node.path,
+                                    '{}.{}'.format(package,
+                                                   directory_node.name))
+
+    def _fill_from_ini(self, datavars, profile_path):
+        ini_filler = NamespaceIniFiller()
+        profile_walker = ProfileWalker(self.basename,
+                                       self.get_repository_map(datavars))
+        for file_path in profile_walker.find(profile_path):
+            try:
+                ini_file_text = read_file(file_path)
+                ini_filler.fill(datavars, ini_file_text)
+            except FilesError:
+                pass
+
+    def get_repository_map(self, datavars):
+        return {repo['name']: repo['path']
+                for repo in datavars.os.gentoo.repositories}
+
+    @contextmanager
+    def test(self, file_name, namespace):
+        print('IMPORT: {}.{}'.format(namespace.get_fullname(), file_name))
+        try:
+            yield self
+        finally:
+            print('IMPORTED FROM: {}.{}'.format(namespace.get_fullname(),
+                                                file_name))
 
 
 class OldVariableLoader:
@@ -422,20 +465,23 @@ class OldVariableLoader:
         return module
 
 
-class ProfileFiller:
+class OldProfileFiller:
     """
     Заполнитель значений переменных из файлов calculate.ini в профилях
     """
     basename = "calculate.ini"
 
-    def get_repository_map(self, namespace):
+    def get_repository_map(self, datavars):
-        return {
+        return {repo['name']: repo['path']
-            x.name.get_value(): x.path.get_value()
+                for repo in datavars.os.gentoo.repositories}
-            for x in namespace.os.gentoo.repositories
+
-        }
+    def fill(self, datavars, profile_path):
-
+        ini_filler = NamespaceIniFiller()
-    def fill(self, namespace, profile_path):
+        profile_walker = ProfileWalker(self.basename,
-        nif = NamespaceIniFillerStrict()
+                                       self.get_repository_map(datavars))
-        pw = ProfileWalker(self.basename, self.get_repository_map(namespace))
+        for file_path in profile_walker.find(profile_path):
-        for fn in pw.find(profile_path):
+            try:
-            nif.fill(namespace, readFile(fn))
+                ini_file_text = read_file(file_path)
+                ini_filler.fill(datavars, ini_file_text)
+            except FilesError:
+                pass

tests/vars/test_alt_vars.py

@@ -586,7 +586,7 @@ class TestNamespace:
             Variable('repositories', type=TableType,
                      source=Dependence('.config', depend=config_source))
 
-        assert datavars.os.repositories.get_table() == value
+        assert datavars.os.repositories == value
 
     # Тестируем теперь заполнитель переменных из calculate.ini
     def test_if_calculate_ini_file_is_used_just_to_create_some_variables__the_namespace_filler_will_create_them(self):
@@ -763,7 +763,7 @@ value = value_2
 """
 
         namespace_filler.fill(datavars, first_ini_text)
-        assert datavars.custom.test.get_table() ==\
+        assert datavars.custom.test ==\
             [{'name': 'name_0', 'value': 'value_0'},
              {'name': 'name_1', 'value': 'value_1'},
              {'name': 'name_2', 'value': 'value_2'}]
@@ -835,9 +835,9 @@ value = weird_value
 """
 
         namespace_filler.fill(datavars, first_ini_text)
-        assert datavars.custom.test.get_table() ==\
+        assert datavars.custom.test ==\
             [{'name': 'name_0', 'value': 'value_0'}]
-        assert datavars.os.test.get_table() ==\
+        assert datavars.os.test ==\
             [{'name': 'strange_name', 'value': 'weird_value'}]
 
     def test_if_calculate_ini_file_is_used_for_modifying_of_the_table_from_calculate_ini_file__the_NamespaceIniFiller_object_modifies_the_table(self):
@@ -857,7 +857,7 @@ value = value_1
 """
 
         namespace_filler.fill(datavars, first_ini_text)
-        assert datavars.custom.test.get_table() ==\
+        assert datavars.custom.test ==\
             [{'name': 'name_0', 'value': 'value_0'},
              {'name': 'name_1', 'value': 'value_1'}]
 
@@ -868,7 +868,7 @@ value = another_value
 """
 
         namespace_filler.fill(datavars, second_ini_text)
-        assert datavars.custom.test.get_table() ==\
+        assert datavars.custom.test ==\
             [{'name': 'name_0', 'value': 'value_0'},
              {'name': 'other_name', 'value': 'another_value'}]
 
@@ -896,7 +896,7 @@ value = another_value
 """
 
         namespace_filler.fill(datavars, first_ini_text)
-        assert datavars.namespace_1.test.get_table() ==\
+        assert datavars.namespace_1.test ==\
             [{'name': 'new_name', 'value': 'other_value'},
              {'name': 'common_name', 'value': 'another_value'},
              {'name': 'name_3', 'value': 'value_3'}]
@@ -910,3 +910,20 @@ value = another_value
 
         assert datavars.main.strange_variable == 'weird_value'
         assert datavars.main.plain_variable is True
+
+    def test_loader_2(self):
+        Namespace.reset()
+        loader = VariableLoader()
+        datavars = loader.load_variables('tests/vars/testfiles/variables_1')
+
+        assert datavars.main.chroot == '/'
+        assert datavars.level.simple == "simple value"
+        assert datavars.level.use_local_simple == "Using simple value"
+        assert datavars.level.use_full_simple == "Using simple value"
+        assert datavars.level.device_child == "hdd"
+        assert datavars.level.device == [{"dev": "/dev/sda",
+                                          "type": "hdd",
+                                          "name": "Samsung SSD"},
+                                         {"dev": "/dev/sdb",
+                                          "type": "flash",
+                                          "name": "Transcend 64GB"}]

tests/vars/testfiles/variables_1/level/__init__.py

@@ -0,0 +1,62 @@
+from calculate.vars.alt_datavars import Namespace, Variable, Dependence,\
+                                        StringType, HashType, TableType,\
+                                        ListType, FloatType
+
+
+Variable('simple', type=StringType, source='simple value')
+
+Variable('use_local_simple', type=StringType,
+         source=Dependence('.simple',
+                           depend=lambda simple: 'Using {}'.format(
+                                                                simple.value)))
+
+Variable('use_full_simple', type=StringType,
+         source=Dependence('level.simple',
+                           depend=lambda simple: 'Using {}'.format(
+                                                                simple.value)))
+
+Variable('disks', type=ListType,
+         source=["/dev/sda1", "/dev/sda2", "/dev/sda3"])
+
+Variable('version', type=FloatType, source='1.0')
+
+Variable('my_shortname', type=StringType, source='CLD')
+
+Variable('linux', type=HashType,
+         source=Dependence('.version', '.my_shortname',
+                           depend=lambda version, my_shortname:
+                           {'version': version.value,
+                            'shortname': my_shortname.value}))
+
+Variable('shortname_test', type=StringType,
+         source=Dependence('.linux.shortname',
+                           depend=lambda shortname: '{} test'.format(
+                                                             shortname.value)))
+
+Variable('device_list', type=ListType,
+         source=["/dev/sda", "/dev/sdb"])
+
+
+def get_device_table(device_list):
+    map_data = {'/dev/sda': ["hdd", "Samsung SSD"],
+                '/dev/sdb': ["flash", "Transcend 64GB"],
+                '/dev/sdc': ["usbhdd", "WD 1TB"]}
+    default_value = ["hdd", "Unknown"]
+    print('device_list = {}'.format(device_list.value))
+    return [{"dev": device,
+             "type": map_data.get(device, default_value)[0],
+             "name": map_data.get(device, default_value)[1]}
+            for device in device_list.value]
+
+
+Variable('device', type=TableType, source=Dependence('.device_list',
+                                                     depend=get_device_table))
+
+Variable('device_child', type=StringType,
+         source=Dependence('.device',
+                           depend=lambda device: device.value[0]['type']))
+
+with Namespace('level_3'):
+    Variable('my_var_1', type=StringType, source='testing')
+
+    Variable('my_var_2', type=StringType, source='testing_2')

tests/vars/testfiles/variables_1/level/level_2/__init__.py

@@ -0,0 +1,7 @@
+from calculate.vars.alt_datavars import Variable, StringType, Dependence
+
+
+Variable('vargetter', type=StringType,
+         source=Dependence('main.chroot',
+                           depend=lambda chroot:
+                           '{} test'.format(chroot.value)))

tests/vars/testfiles/variables_1/main/__init__.py

@@ -0,0 +1,5 @@
+from calculate.vars.alt_datavars import Variable, StringType
+
+
+Variable('chroot', type=StringType, source='/', readonly=True)
+print('chroot really created')

tests/vars/testfiles/variables_1/os/__init__.py

@@ -0,0 +1,36 @@
+from calculate.vars.alt_datavars import Namespace, Variable, Dependence,\
+                                        StringType, HashType, TableType
+
+with Namespace('linux'):
+    Variable('shortname', source='', type=StringType)
+
+    Variable('ver', source='', type=StringType)
+
+    Variable('fullname', source='', type=StringType)
+
+    Variable('subname', source='', type=StringType)
+
+    Variable('arch', source='', type=StringType)
+
+    Variable('test', source='', type=StringType)
+
+    def get_title(subname, fullname, ver):
+        if subname.value:
+            return '{} {} {}'.format(fullname.value, subname.value, ver.value)
+        else:
+            return '{} {}'.format(fullname.value, ver.value)
+    Variable('title', type=StringType,
+             source=Dependence('.subname', '.fullname', '.ver',
+                               depend=get_title))
+
+Variable('hashvar', source={'value1': 'test1',
+                            'value2': 'test2'}, type=HashType)
+
+Variable('calculate', type=StringType,
+         source=lambda hashvar: "{} {}".format(hashvar.value['value1'],
+                                               hashvar.value['value2']))
+
+Variable('tablevar', type=TableType, source=[{"dev": "/dev/sdb1",
+                                              "mount": "/"},
+                                             {"dev": "/dev/sdb2",
+                                              "mount": "/var/calculate"}])