Server development is started. Ability to run templates using scripts is added. fixed #14

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *.pyo
 *.swp
+*.sock
 build/
 dist/
 calculate_lib.egg-info/

calculate/commands/commands.py

@@ -1,14 +1,13 @@
-from types import FunctionType
 from typing import Union, Callable, List
-from calculate.scripts.scripts import Script
-from calculate.variables.parameters import Parameters
-from calculate.variables.datavars import NamespaceNode, DependenceAPI,\
-                                         DependenceError, VariableNotFoundError
-from calculate.variables.loader import Datavars
-from calculate.utils.io_module import IOModule
+from ..scripts.scripts import Script
+from ..parameters.parameters import BaseParameter, Parameters
+from ..variables.datavars import NamespaceNode, DependenceAPI,\
+                                         VariableNotFoundError
+from ..variables.loader import Datavars
+from ..utils.io_module import IOModule
 
 
-class CommandCreationError(Exception):
+class CommandDescriptionError(Exception):
     '''Исключение кидаемое при наличии ошибок во время создания объекта
     команды.'''
     def __init__(self, message: str, command_id: str = '', title: str = ''):
@@ -38,103 +37,182 @@ class CommandInitializationError(Exception):
 
 
 class Command:
-    '''Класс команды, предназначен для хранения информации о команде и привязки ее к модулю ввода/вывода'''
-    def __init__(self, command_id: str = '', category: str = '',
+    '''Класс описания команды, предназначен для хранения информации о команде
+    и создания по данному описанию объекта лаунчера команды.'''
+    def __init__(self, command_id: str = '',
+                 category: str = '',
                  title: str = '',
                  script: Union[Callable, Script, None] = None,
-                 parameters: Union[Parameters, None] = None,
+                 args: Union[tuple, list] = tuple(),
+                 parameters: Union[List[BaseParameter], None] = None,
                  namespace: Union[str, NamespaceNode, None] = None,
-                 command: str = '', gui: bool = False,
+                 command: str = '',
+                 gui: bool = False,
                  icon: Union[str, List[str]] = '',
-                 setvars: dict = {}, rights: List[str] = []):
-        self._datavars: Union[Datavars, NamespaceNode, None] = None
-
+                 setvars: dict = {},
+                 rights: List[str] = []):
+        # Идентификатор команды обязателен.
         if not command_id:
-            raise CommandCreationError('command ID is not set')
-        self._id: str = command_id
+            raise CommandDescriptionError('command ID is not set')
+        self.__id: str = command_id
 
         # Если собственно команда не была указана, получаем ее из ID.
         if command:
-            self._command: str = command
+            self.__command: str = command
         else:
-            self._command: str = f'cl_{self._id}'
+            self.__command: str = f'cl_{self.__id}'
 
+        # Название команды.
         if not title:
-            raise CommandCreationError("title is not set",
-                                       command_id=command_id)
-        self._title: str = title
+            raise CommandDescriptionError("title is not set",
+                                          command_id=command_id)
+        self.__title: str = title
 
+        # Категория, к которой относится команда.
         if not category:
-            raise CommandCreationError("category is not set",
-                                       command_title=title)
-        self._category: str = category
+            raise CommandDescriptionError("category is not set",
+                                          command_title=title)
+        self.__category: str = category
 
         # Пространство имен относительно которого, будет выполняться скрипт.
-        self._namespace: Union[str, NamespaceNode, None] = namespace
+        self.__namespace: Union[str, NamespaceNode, None] = namespace
 
         # Параметры, указываемые при вызове этой команды.
         if parameters is None:
-            raise CommandCreationError("parameters is not set",
-                                       command_title=title)
-        self._parameters: Parameters = parameters
+            raise CommandDescriptionError("parameters is not set",
+                                          command_title=title)
+        self.__parameters: List[BaseParameter] = parameters
 
         # Скрипт выполняемый при вызове этой команды.
         if not script:
-            raise CommandCreationError("script is not set",
-                                       command_title=title)
-        elif isinstance(script, FunctionType):
-            # Поддержка способа описания скрипта в функции.
-            self._script: Script = script()
-        else:
-            self._script: Script = script
+            raise CommandDescriptionError("script is not set",
+                                          command_title=title)
+        self.__script: Script = script
+
+        # Аргументы, с которыми будет запущен скрипт. Пока только статичные
+        # значения.
+        self.__args = args
 
         # Параметр указывающий, должена ли данная команда отображаться в
         # графическом интерфейсе.
-        self._gui: bool = gui
+        self.__gui: bool = gui
 
         # Устанавливаем название иконки.
-        if not icon and self._gui:
-            raise CommandCreationError("icon is not set",
-                                       command_title=title)
-        self._icon: Union[str, List[str]] = icon
+        if not icon and self.__gui:
+            raise CommandDescriptionError("icon is not set",
+                                          command_title=title)
+        self.__icon: Union[str, List[str]] = icon
 
         # Словарь с переменными и значениями, которые нужно им присвоить.
-        self._setvars = setvars
+        self.__setvars: dict = setvars
 
         # Права, необходимые данной команде.
+        # TODO разобраться с тем, как использовать.
         if not rights:
-            raise CommandCreationError("rights is not set",
-                                       title=title)
-        self._rights = rights
+            raise CommandDescriptionError("rights is not set",
+                                          title=title)
+        self.__rights = rights
+
+    def make_runner(self, datavars: Union[Datavars, NamespaceNode],
+                    output: IOModule):
+        '''Метод создающий на основе данного описания команды экземпляр раннера.
+        '''
+        return CommandRunner(self, datavars, output)
+
+    # Интерфейс иммутабельного объекта описания команды.
+    @property
+    def id(self) -> str:
+        return self.__id
+
+    @property
+    def command(self) -> str:
+        return self.__command
+
+    @property
+    def title(self) -> str:
+        return self.__title
+
+    @property
+    def category(self) -> str:
+        return self.__category
+
+    @property
+    def namespace(self) -> str:
+        return self.__namespace
+
+    @property
+    def parameters(self) -> str:
+        return self.__parameters
 
     @property
     def script(self) -> Script:
-        return self._script
+        return self.__script
 
-    def initialize(self, datavars: Union[Datavars, NamespaceNode],
-                   output: IOModule) -> 'Command':
-        '''Метод для инициализации всего, что нужно инициализировать в команде.
-        '''
+    @property
+    def args(self) -> tuple:
+        return self.__args
+
+    @property
+    def gui(self) -> bool:
+        return self.__gui
+
+    @property
+    def icon(self) -> Union[str, List[str]]:
+        return self.__icon
+
+    @property
+    def setvars(self) -> dict:
+        return self.__setvars
+
+    @property
+    def rights(self) -> List[str]:
+        return self.__rights
+
+    def __repr__(self):
+        return f"<Command: {self.__title}>"
+
+
+class CommandRunner:
+    def __init__(self, command: Command,
+                 datavars: Union[Datavars, NamespaceNode],
+                 output: IOModule):
+        '''Класс инкапулирующий все данные о команде, а также необходимые для
+        нее параметры, модуль ввода-вывода, переменные и прочее. Предназначен
+        для конфигурации команды и запуска ее в воркере.'''
+        self._command = command
         self._datavars = datavars
         self._output = output
 
-        # Сначала найдем пространство имен.
-        if self._namespace is not None:
-            if isinstance(self._namespace, str):
-                self._namespace = self._find_namespace(self._namespace,
-                                                       datavars)
-
+        # Ищем пространство имен.
+        if self._command.namespace is not None:
+            if isinstance(self._command.namespace, str):
+                self._namespace = self._find_namespace(
+                                           self._command.namespace,
+                                           datavars)
         # Инициализируем параметры.
-        self._parameters.initialize(datavars)
+        self._parameters = Parameters(self._datavars)
+        self._parameters.add(*self._command.parameters)
+
+        # Получаем запускатель скрипта.
+        self._script_launcher = self._command.script.make_launcher(
+                                                               output,
+                                                               datavars,
+                                                               self._namespace)
+        # Устанавливаем переменные, если нужно.
+        if self._command.setvars:
+            self._set_variables(self._command.setvars,
+                                datavars, self._namespace)
 
-        # Инициализируем скрипт.
-        self._script.initialize(output, datavars, self._namespace)
+    def run_command(self):
+        args = self._command.args
+        self._script_launcher(*args)
 
-        # Устанавливаем переменные, если нужно.
-        if self._setvars:
-            self._set_variables(self._setvars, datavars, self._namespace)
+    def set_parameters(self, values: dict):
+        pass
 
-        return self
+    def get_parameters(self, group: Union[str, None] = None):
+        '''Метод для установки параметров.'''
+        pass
 
     def _find_namespace(self, namespace_path: str,
                         datavars: Union[Datavars, NamespaceNode]

calculate/logging.py

@@ -0,0 +1,22 @@
+from logging import DEBUG
+
+dictLogConfig = {
+    "version": 1,
+    "formatters": {
+        "form_1": {
+            "format":
+            '%(asctime)s %(levelname)s: %(message)s'
+            }
+        },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "form_1",
+            "level": DEBUG
+            }
+        },
+    "root": {
+        "handlers": ['console'],
+        "level": DEBUG
+        }
+}

calculate/variables/parameters.py → calculate/parameters/parameters.py

@@ -1,10 +1,10 @@
 # vim: fileencoding=utf-8
 #
-from .datavars import DependenceAPI, VariableWrapper, NamespaceNode
-from .loader import Datavars
+from ..variables.datavars import DependenceAPI, VariableWrapper, NamespaceNode
+from ..variables.loader import Datavars
 from collections import OrderedDict
 from contextlib import contextmanager
-from typing import Tuple, Union
+from typing import Tuple, Union, Any
 
 
 class ParameterError(Exception):
@@ -541,7 +541,7 @@ class GroupWrapper:
 
 class Parameters:
     '''Класс контейнера параметров.'''
-    def __init__(self, datavars: Datavars = None, check_order=[]):
+    def __init__(self, datavars: Datavars, check_order=[]):
         self._datavars = datavars
         self._parameters = OrderedDict()
         self._validation_dict = OrderedDict()
@@ -558,24 +558,8 @@ class Parameters:
         # параметров.
         self._gui_helpers = {}
 
-        self._initialized = False
-
-    def initialize(self, datavars: Union[Datavars, NamespaceNode]):
-        if not self._initialized:
-            self._datavars = datavars
-            if self._parameters:
-                parameters = self._parameters
-                self._parameters = OrderedDict()
-                self._add(parameters)
-            self._initialized = True
-        return self
-
     def add(self, *parameters: Tuple[BaseParameter]):
         '''Метод для добавления некоторой совокупности параметров.'''
-        if not self._initialized:
-            self._parameters = []
-            self._parameters.extend(parameters)
-
         for parameter in parameters:
             self.add_parameter(parameter)
 
@@ -671,12 +655,12 @@ class Parameters:
             self._validation = False
             self._validation_dict = OrderedDict()
 
-    def get_group_parameters(self, group_name):
+    def get_group_parameters(self, group_name: str):
         '''Метод для получения списка параметров, относящихся к указанной
         группе.'''
         return self._parameters[group_name]
 
-    def get_descriptions(self):
+    def get_descriptions(self) -> dict:
         '''Метод для получения словаря с описанием параметров.'''
         output = OrderedDict()
         for group, parameters in self._parameters.items():
@@ -700,10 +684,10 @@ class Parameters:
         return output
 
     @property
-    def datavars(self):
+    def datavars(self) -> Union[Datavars, NamespaceNode]:
         return self._datavars
 
-    def __getitem__(self, name):
+    def __getitem__(self, name: str) -> Any:
         for group, parameters in self._parameters.items():
             for parameter in parameters:
                 if parameter._name == name or parameter._shortname == name:

calculate/scripts/scripts.py

@@ -3,9 +3,10 @@
 import re
 import inspect
 from typing import Callable, Any, Union, List, Generator
+from calculate.templates.template_processor import DirectoryProcessor
 from calculate.variables.datavars import DependenceAPI, DependenceError,\
                                          VariableNode, NamespaceNode,\
-                                         HashType, TableType
+                                         HashType, TableType, StringType
 from calculate.variables.loader import Datavars
 from calculate.utils.io_module import IOModule
 from collections.abc import Iterable, Mapping
@@ -35,79 +36,112 @@ class ConditionError(Exception):
 
 
 class Script:
-    '''Класс скрипта, собирает задачи, обработчики, блоки и прочее. Принимает
-    аргументы.'''
-    def __init__(self, script_id: str, args: List[Any] = [],
+    '''Класс скрипта, собирает задачи, обработчики, блоки и прочее, хранит их
+    в себе. Создает экземпляры лаунчеров.'''
+    def __init__(self, script_id: str,
+                 args: List[Any] = [],
                  success_message: str = None,
                  failed_message: str = None,
                  interrupt_message: str = None):
         self._id: str = script_id
-        self._items: List[Union['Task', 'Block', 'Handler', 'Run']] = None
+        self.__items: List[Union['Task', 'Block', 'Handler', 'Run']] = None
+        self.__tasks_is_set: bool = False
 
-        self._args_names: list = args
-        self._args_vars: list = []
+        self.__args: list = args
 
-        self._success_message: str = success_message
-        self._failed_message: str = failed_message
-        self._interrupt_message: str = interrupt_message
+        self.__success_message: str = success_message
+        self.__failed_message: str = failed_message
+        self.__interrupt_message: str = interrupt_message
 
-        self._output: IOModule = None
-        self._datavars: Union[Datavars, NamespaceNode] = None
-        self._namespace: Union[NamespaceNode, None] = None
+    @property
+    def id(self) -> str:
+        return self._id
 
-        self._handlers: set = set()
-        self._initialized: bool = False
+    @property
+    def args(self) -> list:
+        return self.__args
+
+    @property
+    def items(self) -> list:
+        return self.__items
+
+    @property
+    def success_message(self) -> str:
+        return self.__success_message
+
+    @property
+    def failed_message(self) -> str:
+        return self.__failed_message
+
+    @property
+    def interrupt_message(self) -> str:
+        return self.__interrupt_message
 
     def tasks(self, *items: List[Union['Task', 'Block', 'Handler', 'Run']]
               ) -> 'Script':
         '''Метод для указания задач и контейнеров, из которых состоит скрипт.
         '''
-        self._items = items
+        if self.__tasks_is_set:
+            raise ScriptError("Script object is immutable.")
+
+        self.__items = items
+        self.__tasks_is_set = bool(self.__items)
         return self
 
-    def initialize(self, output: IOModule,
-                   datavars: Union[Datavars, NamespaceNode],
-                   namespace: NamespaceNode = None) -> 'Script':
+    def make_launcher(self, output: IOModule,
+                      datavars: Union[Datavars, NamespaceNode],
+                      namespace: NamespaceNode = None) -> 'ScriptLauncher':
+        '''Метод для создания экземпляра объекта, для запуска данного
+        скрипта.'''
+        return ScriptLauncher(self, output, datavars, namespace)
+
+
+class ScriptLauncher:
+    def __init__(self, script: Script,
+                 output: IOModule,
+                 datavars: Union[Datavars, NamespaceNode],
+                 namespace: NamespaceNode):
         '''Метод для инициализации, состоящей в установке модулей вывода,
         модуля переменных, текущего пространства имен, а также создании
         переменных скрипта.'''
+        self._script = script
+
         self._output = output
         self._datavars = datavars
         self._namespace = namespace
 
-        if not self._initialized:
-            self._script_namespace, self._args_vars =\
-                self.make_script_variables(self._id,
-                                           self._args_names,
-                                           self._datavars)
-            self._initialized = True
-        return self
+        self._script_namespace, self._args_vars = self.make_script_variables(
+                                                                   script.id,
+                                                                   script.args,
+                                                                   datavars)
 
     def run(self) -> None:
         '''Метод для запуска задач, содержащихся в данном скрипте.'''
         essential_error = None
-        handlers_to_run = []
+        founded_handlers = []
+        handlers_to_run = set()
 
-        for task_item in self._items:
+        for task_item in self._script.items:
             if isinstance(task_item, Handler):
-                if task_item.id in self._handlers:
-                    handlers_to_run.append(task_item)
+                founded_handlers.append(task_item)
             elif essential_error is None:
                 try:
                     output = task_item.run(self._output,
                                            self._datavars,
                                            self._script_namespace,
                                            self._namespace)
-                    self._handlers.update(output)
+                    handlers_to_run.update(output)
                 except Exception as error:
                     if isinstance(error, TaskError):
-                        self._handlers.update(error.handlers)
+                        handlers_to_run.update(error.handlers)
                     essential_error = error
 
-        for handler in handlers_to_run:
+        for handler in founded_handlers:
+            if handler.id not in handlers_to_run:
+                continue
             try:
-                task_item.run(self._output, self._datavars,
-                              self._script_namespace, self._namespace)
+                handler.run(self._output, self._datavars,
+                            self._script_namespace, self._namespace)
             except Exception as error:
                 # TODO Разобраться что делать с ошибками в обработчике.
                 self._output.set_error(f"error in handler '{task_item.id}':"
@@ -123,7 +157,7 @@ class Script:
         '''Метод для создания переменных скрипта. Создает пространство имен
         скрипта и переменных аргументов.'''
         if 'scripts' not in datavars:
-            scripts = Script.make_scripts_namespace(datavars)
+            scripts = ScriptLauncher.make_scripts_namespace(datavars)
         else:
             scripts = datavars.scripts
 
@@ -132,11 +166,14 @@ class Script:
             scripts.add_namespace(current_script)
         else:
             current_script = scripts[script_id]
-            current_script.clear()
+            # current_script.clear()
 
         args_vars = []
         for arg in args:
-            args_vars.append(VariableNode(arg, current_script))
+            if arg not in current_script.variables:
+                args_vars.append(VariableNode(arg, current_script))
+            else:
+                args_vars.append(current_script.variables[arg])
         return current_script, args_vars
 
     @staticmethod
@@ -157,14 +194,15 @@ class Script:
 
     def __call__(self, *args):
         '''Метод для запуска скрипта с аргументами.'''
-        if len(args) < len(self._args_names):
-            raise ScriptError((f"script '{self._id}' missing"
-                               f" {len(self._args_names) - len(args)} required"
-                               " arguments: '") +
-                              "', '".join(self._args_names[len(args):]) + "'")
-        elif len(args) > len(self._args_names):
-            raise ScriptError(f"script '{self._id}' takes"
-                              f" {len(self._args_names)} arguments, but"
+        if len(args) < len(self._script.args):
+            raise ScriptError(
+                             (f"script '{self._script.id}' missing"
+                              f" {len(self._script.args) - len(args)} required"
+                              " arguments: '") +
+                              "', '".join(self._script.args[len(args):]) + "'")
+        elif len(args) > len(self._script.args):
+            raise ScriptError(f"script '{self._script.id}' takes"
+                              f" {len(self._script.args)} arguments, but"
                               f" {len(args)} were given")
 
         args_values = self._get_args_values(args, self._datavars,
@@ -191,10 +229,106 @@ class Script:
         return args_values
 
 
+class RunTemplate:
+    '''Класс запускателя наложения шаблонов.'''
+    def __init__(self, id: str = '',
+                 action: str = '',
+                 package: str = '',
+                 chroot_path: str = '',
+                 root_path: str = '',
+                 dbpkg: bool = True,
+                 essential: bool = True,
+                 **group_packages):
+        self._id: str = id
+        self._action: str = action
+        self._package: str = package or None
+        self._chroot_path: str = chroot_path or None
+        self._root_path: str = root_path or None
+
+        self._dbpkg = dbpkg
+        self._essential = essential
+        self._groups: dict = group_packages
+
+    def _initialize(self, output: IOModule,
+                    datavars: Union[Datavars, NamespaceNode]):
+        # Проверяем наличие идентификатора.
+        if not self._id:
+            error_msg = "id is not set for templates runner"
+            if self._essential:
+                raise TaskError(error_msg)
+            else:
+                output.set_error(error_msg)
+                return set()
+
+        # Проверяем наличие параметра action, который обязателен.
+        if not self._action:
+            error_msg = ("action parameter is not set for templates runner"
+                         f" '{self._id}'")
+            if self._essential:
+                raise TaskError(error_msg)
+            else:
+                output.set_error(error_msg)
+                return set()
+
+        # Если установлен chroot_path -- устанавливаем значение соответствующей
+        # переменной.
+        if self._chroot_path is not None:
+            if 'cl_chroot_path' in datavars.main:
+                datavars.main['cl_chroot_path'].source = self._chroot_path
+            else:
+                VariableNode("cl_chroot_path", datavars.main,
+                             variable_type=StringType,
+                             source=self._chroot_path)
+
+        # Если установлен root_path -- устанавливаем значение соответствующей
+        # переменной.
+        if self._root_path is not None:
+            if 'cl_root_path' in datavars.main:
+                datavars.main['cl_root_path'].source = self._root_path
+            else:
+                VariableNode("cl_root_path", datavars.main,
+                             variable_type=StringType,
+                             source=self._chroot_path)
+
+        if self._groups:
+            groups = list(self._groups.keys())
+            for group, atoms in groups:
+                if isinstance(atoms, str):
+                    self._groups[group] = [name.strip() for name
+                                           in atoms.split(',')]
+
+    def run(self, output: IOModule,
+            datavars: Union[Datavars, NamespaceNode],
+            script_namespace: NamespaceNode,
+            namespace: NamespaceNode) -> set:
+        '''Метод для запуска наложения шаблонов.'''
+        self._initialize(output, datavars)
+        template_processor = DirectoryProcessor(self._action,
+                                                datavars_module=datavars,
+                                                package=self._package,
+                                                output_module=output,
+                                                dbpkg=self._dbpkg,
+                                                **self._groups)
+        changed_files = template_processor.process_template_directories()
+        self._create_result_var(script_namespace,
+                                changed_files=changed_files)
+        return set()
+
+    def _create_result_var(self, script_namespace: NamespaceNode,
+                           changed_files: dict = {},
+                           skipped: list = []) -> None:
+        '''Метод для создания переменной задачи, в которую отправляются
+        результаты вычислений, имеющихся в функции action.'''
+        VariableNode(self._id, script_namespace, variable_type=HashType,
+                     source={"changed": changed_files,
+                             "skipped": skipped})
+
+
 class Run:
     '''Класс запускателя скриптов.'''
     def __init__(self, script: Script, namespace: str = None,
-                 args: List[Any] = [], when: 'Var' = None,
+                 args: List[Any] = [],
+                 when: 'Var' = None,
                  essential: bool = True):
         self._script: Script = script
         self._namespace: str = namespace
@@ -204,9 +338,11 @@ class Run:
 
         self._essential = essential
 
-    def run(self, output: IOModule, datavars: Union[Datavars, NamespaceNode],
-            parent_namespace: NamespaceNode, namespace: NamespaceNode) -> set:
-        '''Метод для запуска скрипта, указанного имеющегося в запускателе.'''
+    def run(self, output: IOModule,
+            datavars: Union[Datavars, NamespaceNode],
+            parent_namespace: NamespaceNode,
+            namespace: NamespaceNode) -> set:
+        '''Метод для запуска скрипта, указанного в запускателе.'''
         if self._condition is None or self._condition(datavars,
                                                       namespace,
                                                       parent_namespace):
@@ -222,8 +358,10 @@ class Run:
 
             try:
                 # if not self._script._initialized:
-                self._script.initialize(output, datavars, self._namespace)
-                self._script(*self._args)
+                launcher = self._script.make_launcher(
+                                                     output, datavars,
+                                                     namespace=self._namespace)
+                launcher(*self._args)
             except (ScriptError, TaskError) as error:
                 if self._essential:
                     raise ScriptError(f"essential script '{self._script._id}'"
@@ -330,8 +468,6 @@ class Task:
         else:
             self._args = self._update_arguments(self._args_sources)
 
-        return self
-
     def run(self, output: IOModule,
             datavars: Union[Datavars, NamespaceNode],
             script_namespace: NamespaceNode,
@@ -664,7 +800,8 @@ class Until(Loop):
 
 class Block:
     '''Класс блока задач.'''
-    def __init__(self, *tasks: List[Task], when: Union['Var', None] = None,
+    def __init__(self, *tasks: List[Task],
+                 when: Union['Var', None] = None,
                  loop: Loop = Loop()):
         self._tasks: List[Task] = tasks
         self._rescue_tasks: List[Task] = None
@@ -736,7 +873,8 @@ class Handler:
     def id(self) -> str:
         return self._id
 
-    def run(self, output: IOModule, datavars: Union[Datavars, NamespaceNode],
+    def run(self, output: IOModule,
+            datavars: Union[Datavars, NamespaceNode],
             script_namespace: NamespaceNode,
             namespace: NamespaceNode) -> None:
         '''Метод для запуска выполнения задач, содержащихся в обработчике.'''

calculate/server/server.py

@@ -0,0 +1,155 @@
+from ..variables.loader import Datavars
+from ..commands.commands import Command
+from ..logging import dictLogConfig
+from logging.config import dictConfig
+from logging import getLogger
+from typing import Callable
+from fastapi import FastAPI
+from .worker import Worker
+import importlib
+import uvicorn
+import asyncio
+import os
+
+
+# TODO
+# 1. Разобраться с описанием команд как ресурсов и со всем, что от них зависит.
+# 2. Разобраться с объектами воркеров. И способом их функционирования.
+
+
+class Server:
+    def __init__(self, socket_path='./input.sock',
+                 datavars_path: str = 'calculate/vars/',
+                 commands_path: str = 'calculate/commands'):
+        self._app = FastAPI()
+        self._socket_path = socket_path
+        self._event_loop = asyncio.get_event_loop()
+
+        # Конфигурируем логгирование.
+        dictConfig(dictLogConfig)
+        self._logger = getLogger("main")
+        self.log_msg = {'DEBUG': self._logger.debug,
+                        'INFO': self._logger.info,
+                        'WARNING': self._logger.warning,
+                        'ERROR': self._logger.error,
+                        'CRITICAL': self._logger.critical}
+
+        self._datavars = Datavars(variables_path=datavars_path,
+                                  logger=self._logger)
+
+        # Словарь описаний команд.
+        self._commands = self._get_commands_list(commands_path)
+
+        # Словарь CID и экземпляров команд, передаваемых воркерам.
+        self._commands_instances = {}
+
+        # Словарь WID и экземпляров процессов-воркеров, передаваемых воркерам.
+        self._workers = {}
+
+        # Соответствие путей обработчикам запросов для HTTP-метода GET.
+        self._add_routes(self._app.get,
+                         {"/": self._get_root,
+                          "/commands": self._get_commands,
+                          "/commands/{cid}": self._get_command,
+                          "/workers/{wid}": self._get_worker})
+        self._add_routes(self._app.post,
+                         {"/commands/{command_id}": self._post_command})
+
+    def _get_commands_list(self, commands_path: str) -> list:
+        '''Метод для получения совокупности описаний команд.'''
+        output = {}
+        package = ".".join(commands_path.split("/"))
+
+        for entry in os.scandir(commands_path):
+            if (not entry.name.endswith('.py')
+               or entry.name in {"commands.py", "__init__.py"}):
+                continue
+            module_name = entry.name[:-3]
+            try:
+                module = importlib.import_module("{}.{}".format(package,
+                                                                module_name))
+                for obj in dir(module):
+                    if type(module.__getattribute__(obj)) == Command:
+                        command_object = module.__getattribute__(obj)
+                        output[command_object.id] = command_object
+            except Exception:
+                continue
+        return output
+
+    async def _get_root(self) -> dict:
+        '''Обработчик корневых запросов.'''
+        return {'msg': 'root msg'}
+
+    async def _get_commands(self) -> dict:
+        '''Обработчик, отвечающий на запросы списка команд.'''
+        response = {}
+        for command_id, command_object in self._commands.items():
+            response.update({command_id: {"title": command_object.title,
+                                          "category": command_object.category,
+                                          "icon": command_object.icon,
+                                          "command": command_object.command}})
+        return response
+
+    async def _get_command(self, cid: int) -> dict:
+        '''Обработчик запросов списка команд.'''
+        if cid not in self._commands_instances:
+            # TODO добавить какую-то обработку ошибки.
+            pass
+        return {'id': cid,
+                'name': f'command_{cid}'}
+
+    async def _get_worker(self, wid: int):
+        '''Тестовый '''
+        self._make_worker(wid=wid)
+        worker = self._workers[wid]
+        worker.run(None)
+        await worker.send({"text": "INFO"})
+        data = await worker.get()
+        if data['type'] == 'log':
+            self.log_msg[data['level']](data['msg'])
+        return data
+
+    async def _post_command(self, command_id: str) -> int:
+        if command_id not in self._commands:
+            # TODO добавить какую-то обработку ошибки.
+            pass
+        return
+
+    def _add_routes(self, method: Callable, routes: dict) -> None:
+        '''Метод для добавления методов.'''
+        for path, handler in routes.items():
+            router = method(path)
+            router(handler)
+
+    def _make_worker(self, wid: int = None):
+        '''Метод для создания воркера для команды.'''
+        worker = Worker(self._event_loop)
+        if wid is None:
+            self._workers[wid] = worker
+            return wid
+        elif not self._workers:
+            self._workers[0] = worker
+            return 0
+        else:
+            wid = max(self._workers.keys()) + 1
+            self._workers[wid] = worker
+            return wid
+
+    def _make_command(self, command_id: str) -> int:
+        '''Метод для создания команды по ее описанию.'''
+        command_description = self._commands[command]
+
+    @property
+    def app(self):
+        return self._app
+
+    def run(self):
+        '''Метод для запуска сервера.'''
+        # Выгружаем список команд.
+        uvicorn.run(self._app,
+                    uds=self._socket_path)
+
+
+if __name__ == '__main__':
+    server = Server()
+    server.run()

calculate/server/worker.py

@@ -0,0 +1,38 @@
+from multiprocessing import Queue, Process
+# from time import sleep
+
+
+class Worker:
+    def __init__(self, loop):
+        self._output_queue = Queue()
+        self._in_queue = Queue()
+        self._event_loop = loop
+
+    async def send(self, data: dict):
+        self._in_queue.put(data)
+
+    async def get(self):
+        data = await self._event_loop.run_in_executor(None, self._get_output,
+                                                      self._output_queue)
+        return data
+
+    @staticmethod
+    def _get_output(output_queue: Queue) -> dict:
+        return output_queue.get()
+
+    @staticmethod
+    def _main_loop(command, in_queue: Queue, out_queue: Queue):
+        data = in_queue.get()
+        print('\nworker for command:', command)
+        output = {"type": "log",
+                  "level": "INFO",
+                  "msg": f"recieved message {data['text']}"}
+        out_queue.put(output)
+
+    def run(self, command):
+        '''Метод для запуска процесса воркера с заданным '''
+        worker_process = Process(target=self._main_loop,
+                                 args=(command,
+                                       self._in_queue,
+                                       self._output_queue))
+        worker_process.start()

calculate/templates/template_engine.py

@@ -429,8 +429,6 @@ class ParametersProcessor:
             real_path = parameter_value
 
         # Ставим True, чтобы потом проверить этот параметр в postparse
-        print(f'source value {parameter_value}')
-        print(f'source <- {real_path}')
         if not os.path.exists(real_path):
             return True
 
@@ -592,10 +590,6 @@ class ParametersProcessor:
         else:
             groups = self._parameters_container.group
 
-        print(f'PACKAGE: {parameter_value}')
-        print(f'PARSED: {package_atom}')
-        print(f'GROUPS: {groups}')
-        print(f'UNINSTALL: {self._groups.get("uninstall", None)}')
         for group in groups:
             if group == 'install':
                 try:
@@ -610,7 +604,6 @@ class ParametersProcessor:
                 self._parameters_container.remove_parameter('package')
                 return
 
-        print('check failed')
         raise ConditionFailed(f"package '{parameter_value}'"
                               " does not match the template condition",
                               self.lineno)
@@ -623,8 +616,6 @@ class ParametersProcessor:
                 if package[parameter] is not None:
                     if (group_package[parameter] is None
                        or group_package[parameter] != package[parameter]):
-                        print(f'{group_package[parameter]} !='
-                              f' {package[parameter]}')
                         break
             else:
                 if package['use_flags'] is not None:

calculate/templates/template_processor.py

@@ -719,8 +719,6 @@ class TemplateExecutor:
         self.executor_output = {'target_path': None,
                                 'stdout': None,
                                 'stderr': None}
-        print('TARGET PATH:', target_path)
-
         if parameters.append == 'skip':
             return self.executor_output
 
@@ -1762,7 +1760,8 @@ class DirectoryTree:
 class DirectoryProcessor:
     '''Класс обработчика директорий шаблонов.'''
     def __init__(self, action: str, datavars_module=Variables(), package='',
-                 output_module=IOModule(), dbpkg=True, **groups):
+                 output_module=IOModule(), dbpkg=True,
+                 namespace: NamespaceNode = None, **groups):
         if isinstance(action, list):
             self.action = action
         else:
@@ -1781,7 +1780,6 @@ class DirectoryProcessor:
         if 'cl_root_path' in datavars_module.main:
             self.templates_root = join_paths(self.cl_chroot_path,
                                              datavars_module.main.cl_root_path)
-            print('ROOT PATH:', self.templates_root)
         else:
             self.templates_root = self.cl_chroot_path
 
@@ -1826,7 +1824,11 @@ class DirectoryProcessor:
         # Разбираем atom имена пакетов, указанных для групп пакетов.
         if groups:
             for group_name, package_name in groups.items():
-                self._add_package_to_group(group_name, package_name)
+                if isinstance(package_name, list):
+                    for atom in package_name:
+                        self._add_package_to_group(group_name, atom)
+                else:
+                    self._add_package_to_group(group_name, package_name)
 
         # Инициализируем шаблонизатор.
         self.template_engine = TemplateEngine(
@@ -1998,6 +2000,7 @@ class DirectoryProcessor:
             self._run_exec_files()
 
         self.template_executor.save_changes()
+        return self.template_executor.changed_files
 
     def _execute_handlers(self):
         '''Метод для запуска обработчиков добавленных в очередь обработчиков
@@ -2159,7 +2162,6 @@ class DirectoryProcessor:
         необходимости, заполнения деревьев директорий шаблонов, с помощью
         которых далее выполняются шаблоны пакетов из merge.'''
         directory_name = os.path.basename(current_directory_path)
-        print('CURRENT TARGET PATH =', current_target_path)
 
         # Если включено заполнение дерева создаем пустой словарь для сбора
         # содержимого текущей директории.

calculate/variables/loader.py

@@ -1,6 +1,7 @@
 # vim: fileencoding=utf-8
 #
 import os
+import logging
 import importlib
 import importlib.util
 from jinja2 import Environment, FileSystemLoader
@@ -12,7 +13,6 @@ from calculate.variables.datavars import NamespaceNode, VariableNode,\
 from calculate.utils.gentoo import ProfileWalker
 from calculate.utils.files import read_file, FilesError
 from calculate.utils.tools import Singleton
-from calculate.utils.io_module import IOModule
 from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
                       empty, printables, OneOrMore, lineno, line, SkipTo,\
                       LineEnd, Combine, nums
@@ -489,7 +489,7 @@ class VariableLoader:
 
     def __init__(self, datavars, variables_path, repository_map=None):
         self.datavars = datavars
-        self.output = datavars.output
+        self.logger = datavars.logger
         self.ini_filler = NamespaceIniFiller()
 
         self.variables_path = variables_path
@@ -519,9 +519,9 @@ class VariableLoader:
                 if section in current_namespace:
                     current_namespace = current_namespace[section]
                 else:
-                    self.output.set_error("Variable 'os.gentoo.repositories'"
-                                          " is not found. Can not load profile"
-                                          " variables.")
+                    self.logger.error("Variable 'os.gentoo.repositories'"
+                                      " is not found. Can not load profile"
+                                      " variables.")
                     return
             self.repository_map = self._get_repository_map(self.datavars)
 
@@ -530,12 +530,12 @@ class VariableLoader:
            'path' in self.datavars.os.gentoo.profile):
             profile_path = self.datavars.os.gentoo.profile.path
         else:
-            self.output.set_error("Variable 'os.gentoo.profile.path'"
-                                  " is not found. Can not load profile"
-                                  " variables.")
+            self.logger.error("Variable 'os.gentoo.profile.path'"
+                              " is not found. Can not load profile"
+                              " variables.")
             return
 
-        self.output.set_info("Load variables from profile: '{}'.".format(
+        self.logger.info("Load variables from profile: '{}'.".format(
                                                                  profile_path))
         self._fill_from_profile_ini(profile_path)
 
@@ -546,20 +546,20 @@ class VariableLoader:
             env_order = self.datavars.system.env_order
             env_path = self.datavars.system.env_path
         except VariableNotFoundError as error:
-            self.output.set_warning("Can not load additional variables: {}".
-                                    format(str(error)))
+            self.logger.warning("Can not load additional variables: {}".
+                                format(str(error)))
             return
 
         for ini_file in env_order:
-            self.output.set_info("Loading variables from file: '{}'".format(
+            self.logger.info("Loading variables from file: '{}'".format(
                                                                      ini_file))
             if ini_file in env_path:
                 self.fill_from_custom_ini(env_path[ini_file].value)
-                self.output.set_success("Variables from '{}' are loaded".
-                                        format(ini_file))
+                self.logger.info("Variables from '{}' are loaded".format(
+                                                                     ini_file))
             else:
-                self.output.set_warning("File '{}' is not found. Variables are"
-                                        " not loaded".format(ini_file))
+                self.logger.warning("File '{}' is not found. Variables are"
+                                    " not loaded".format(ini_file))
 
     def _fill_from_package(self, current_namespace: NamespaceNode,
                            directory_path: str, package: str) -> None:
@@ -575,6 +575,8 @@ class VariableLoader:
 
         # Сначала загружаем переменные из файлов.
         for file_node in file_nodes:
+            if not file_node.name.endswith('.py'):
+                continue
             file_name = file_node.name[:-3]
             Namespace.set_current_namespace(current_namespace)
             # with self.test(file_name, current_namespace):
@@ -603,8 +605,8 @@ class VariableLoader:
                 ini_file_text = read_file(file_path)
                 self.ini_filler.fill(self.datavars, ini_file_text)
             except FilesError:
-                self.output.set_error("Can not load profile variables from"
-                                      " unexisting file: {}".format(file_path))
+                self.logger.error("Can not load profile variables from"
+                                  " unexisting file: {}".format(file_path))
 
     def _get_repository_map(self, datavars):
         '''Метод для получения из переменной словаря с репозиториями и путями
@@ -621,13 +623,13 @@ class VariableLoader:
             parsing_errors = self.ini_filler.errors
             if parsing_errors:
                 for error in parsing_errors:
-                    self.output.set_error(error)
-                self.output.set_warning('Some variables was not loaded.')
+                    self.logger.error(error)
+                self.logger.warning('Some variables was not loaded.')
             else:
-                self.output.set_success('All variables are loaded.')
+                self.logger.info('All variables are loaded.')
         else:
-            self.output.set_error("Variables are not loaded. File '{}' does"
-                                  " not exist.".format(file_path))
+            self.logger.error("Variables are not loaded. File '{}' does"
+                              " not exist.".format(file_path))
 
     @contextmanager
     def test(self, file_name, namespace):
@@ -704,10 +706,16 @@ class CalculateIniSaver:
 class Datavars:
     '''Класс для хранения переменных и управления ими.'''
     def __init__(self, variables_path='calculate/vars', repository_map=None,
-                 io_module=IOModule()):
+                 logger=None):
         self._variables_path = variables_path
         self._available_packages = self._get_available_packages()
-        self.output = io_module
+        if logger is not None:
+            self.logger = logger
+        else:
+            logger = logging.getLogger("main")
+            # stream_handler = logging.StreamHandler()
+            # logger.addHandler(stream_handler)
+            self.logger = logger
 
         self.root = NamespaceNode('<root>')
         self._loader = VariableLoader(self, self._variables_path,
@@ -749,8 +757,7 @@ class Datavars:
 
     def _load_package(self, package_name):
         '''Метод для загрузки переменных содержащихся в указанном пакете.'''
-        self.output.set_info("Loading datavars package '{}'".format(
-                                                                 package_name))
+        self.logger.info("Loading datavars package '{}'".format(package_name))
         try:
             self._loader.load_variables_package(package_name)
         except Exception as error:
@@ -843,5 +850,5 @@ class Datavars:
                 dict_to_save = self.variables_to_save[target]
                 target_path = target_paths[target].value
                 saver.save_to_ini(target_path, dict_to_save)
-                self.output.set_info("Variables for '{}' is saved in the"
-                                     " file: {}".format(target, target_path))
+                self.logger.info("Variables for '{}' is saved in the"
+                                 " file: {}".format(target, target_path))

client.py

@@ -0,0 +1,33 @@
+import asyncio
+import aiohttp
+
+
+async def get_root(client):
+    async with client.get('http://localhost/') as resp:
+        assert resp.status == 200
+        return await resp.json()
+
+
+async def main():
+    unix_conn = aiohttp.UnixConnector(path='./input.sock')
+
+    try:
+        async with aiohttp.ClientSession(connector=unix_conn) as client:
+            print('---------------------------')
+            print('Request GET "/" HTTP/1.1:')
+            response = await get_root(client)
+            print(f'Response: {response}')
+            print('---------------------------')
+    except KeyboardInterrupt:
+        raise
+    finally:
+        print('\nClosing connection...')
+        await unix_conn.close()
+
+
+if __name__ == '__main__':
+    loop = asyncio.new_event_loop()
+    try:
+        loop.run_until_complete(main())
+    except KeyboardInterrupt:
+        loop.close()

pytest.ini

@@ -36,5 +36,6 @@ markers =
     vars: marker for testing of the datavars module.
     parameters: marker for testing of the parameters module.
 
-    tasks: marker for testing of the tasks.
+    scripts: marker for testing of the scripts.
     commands: marker for testing of the commands.
+    server: marker for testing of the server.

run_server.py

@@ -0,0 +1,6 @@
+from calculate.server.server import Server
+
+
+if __name__ == "__main__":
+    server = Server()
+    server.run()

run_templates.py

@@ -40,7 +40,8 @@ def main():
                                             datavars_module=datavars,
                                             package=package,
                                             output_module=io_module,
-                                            dbpkg=args.dbpkg)
+                                            dbpkg=args.dbpkg,
+                                            **group_packages)
     template_processor.process_template_directories()
 
 

tests/commands/parameters.py

@@ -0,0 +1,39 @@
+from calculate.parameters.parameters import BaseParameter, Integer,\
+                                            String, ValidationError
+
+
+class MyShinyParameter(BaseParameter):
+    type = Integer()
+
+    def validate(self, var):
+        if var.value < 10:
+            raise ValidationError("The value must be greater than 10")
+
+    def bind_method(self, var):
+        return var.value, None
+
+
+class AnotherParameter(BaseParameter):
+    type = String()
+
+    def bind_method(self):
+        return 'default string', None
+
+    def validate(self, value):
+        if not value.startswith('/var/lib'):
+            raise ValidationError("The value must starts with a"
+                                  " '/var/lib'")
+
+
+class OneMoreParameter(BaseParameter):
+    type = String()
+
+    def bind_method(self):
+        return 'default string', None
+
+    def validate(self, value):
+        available_set = {'mystery', 'horror', 'weird'}
+        if value not in available_set:
+            raise ValidationError(f"The value '{value}' is not in"
+                                  f" available. Available values:"
+                                  f" {', '.join(available_set)}")

tests/commands/scripts.py

@@ -0,0 +1,14 @@
+from calculate.scripts.scripts import Script, Task
+
+
+def action(output, arg1: str) -> str:
+    '''Задача выполняемая скриптом ниже.'''
+    return f'os.calculate = {arg1}'
+
+
+# Тестовый скрипт.
+test_script = Script('test_script'
+                     ).tasks(Task(id='task',
+                                  name="Task",
+                                  action=action,
+                                  args=["os.calculate"]))

tests/commands/test_commands.py

@@ -1,14 +1,15 @@
 import os
 import shutil
 import pytest
+
 from calculate.commands.commands import Command
-from calculate.scripts.scripts import Script, Task
+from calculate.parameters.parameters import Description
 from calculate.variables.loader import Datavars
-from calculate.variables.parameters import Parameters, BaseParameter, Integer,\
-                                           String, ValidationError,\
-                                           Description
 from calculate.utils.io_module import IOModule
 
+from .parameters import MyShinyParameter, OneMoreParameter, AnotherParameter
+from .scripts import test_script
+
 
 TESTFILES_PATH = os.path.join(os.getcwd(), 'tests/commands/testfiles')
 
@@ -24,78 +25,37 @@ class TestCommands:
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
                                                         'variables'))
 
-        # Скрипт выполняемый командой.
-        def action(output: IOModule, arg1: str) -> str:
-            return f'os.calculate = {arg1}'
-
-        # Для обеспечения многопоточности вероятно придется делать так.
-        def test_script():
-            return Script('test_script'
-                          ).tasks(Task(id='task',
-                                       name="Task",
-                                       action=action,
-                                       args=["os.calculate"])
-                                  )
-
-        # Параметры данной команды.
-        test_parameters = Parameters().initialize(datavars)
-
-        class MyShinyParameter(BaseParameter):
-            type = Integer()
-
-            def validate(self, var):
-                if var.value < 10:
-                    raise ValidationError("The value must be greater than 10")
-
-            def bind_method(self, var):
-                return var.value, None
-
-        class AnotherParameter(BaseParameter):
-            type = String()
-
-            def bind_method(self):
-                return 'default string', None
-
-            def validate(self, value):
-                if not value.startswith('/var/lib'):
-                    raise ValidationError("The value must starts with a"
-                                          " '/var/lib'")
-
-        class OneMoreParameter(BaseParameter):
-            type = String()
-
-            def bind_method(self):
-                return 'default string', None
-
-            def validate(self, value):
-                available_set = {'mystery', 'horror', 'weird'}
-                if value not in available_set:
-                    raise ValidationError(f"The value '{value}' is not in"
-                                          f" available. Available values:"
-                                          f" {', '.join(available_set)}")
-
-        test_parameters.add(
-            MyShinyParameter('my-shiny', 'The Shiniest ones',
-                             Description(
-                                   short='The shiny parameter',
-                                   full='The shiniest thing known to science.',
-                                   usage='-S COUNT, --my-shiny COUNT'),
-                             shortname='S').bind('os.linux.test_5'),
-            AnotherParameter('my-other', 'The Others',
-                             Description(short='Not from this world.'),
-                             shortname='o'),
-            OneMoreParameter('one-more', 'The Others',
-                             Description(full='Plan 9 from outer space.')))
-
         command = Command(command_id='test',
                           category='Test Category',
                           title='Test',
-                          script=test_script(),
-                          parameters=test_parameters,
+                          script=test_script,
                           namespace='os',
-                          rights=['group'])
-        command.initialize(datavars, IOModule())
-        command.script.run()
+                          rights=['group'],
+                          parameters=[
+                              MyShinyParameter
+                              (
+                                'my-shiny', 'The Shiniest ones',
+                                Description(
+                                  short='The shiny parameter',
+                                  full='The shiniest thing known to science.',
+                                  usage='-S COUNT, --my-shiny COUNT'),
+                                shortname='S'
+                              ).bind('os.linux.test_5'),
+                              AnotherParameter
+                              (
+                                'my-other', 'The Others',
+                                Description(
+                                  short='Not from this world.'),
+                                shortname='o'
+                              ),
+                              OneMoreParameter
+                              (
+                                'one-more', 'The Others',
+                                Description(full='Plan 9 from outer space.')
+                              )])
+
+        command_runner = command.make_runner(datavars, IOModule())
+        command_runner.run_command()
 
         assert datavars.scripts.test_script.task.result ==\
             'os.calculate = test1 test2'
@@ -104,57 +64,7 @@ class TestCommands:
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
                                                         'variables'))
 
-        # Скрипт выполняемый командой.
-        def action(output: IOModule, arg1: str) -> str:
-            return f'os.calculate = {arg1}'
-
-        # Для обеспечения многопоточности вероятно придется делать так.
-        def test_script():
-            return Script('test_script'
-                          ).tasks(Task(id='task',
-                                       name="Task",
-                                       action=action,
-                                       args=["os.calculate"])
-                                  )
-
-        # Параметры данной команды.
-        test_parameters = Parameters().initialize(datavars)
-
-        class MyShinyParameter(BaseParameter):
-            type = Integer()
-
-            def validate(self, var):
-                if var.value < 10:
-                    raise ValidationError("The value must be greater than 10")
-
-            def bind_method(self, var):
-                return var.value, None
-
-        class AnotherParameter(BaseParameter):
-            type = String()
-
-            def bind_method(self):
-                return 'default string', None
-
-            def validate(self, value):
-                if not value.startswith('/var/lib'):
-                    raise ValidationError("The value must starts with a"
-                                          " '/var/lib'")
-
-        class OneMoreParameter(BaseParameter):
-            type = String()
-
-            def bind_method(self):
-                return 'default string', None
-
-            def validate(self, value):
-                available_set = {'mystery', 'horror', 'weird'}
-                if value not in available_set:
-                    raise ValidationError(f"The value '{value}' is not in"
-                                          f" available. Available values:"
-                                          f" {', '.join(available_set)}")
-
-        test_parameters.add(
+        test_parameters = [
             MyShinyParameter('my-shiny', 'The Shiniest ones',
                              Description(
                                    short='The shiny parameter',
@@ -165,23 +75,24 @@ class TestCommands:
                              Description(short='Not from this world.'),
                              shortname='o'),
             OneMoreParameter('one-more', 'The Others',
-                             Description(full='Plan 9 from outer space.')))
+                             Description(full='Plan 9 from outer space.'))]
 
         command = Command(command_id='test',
                           category='Test Category',
                           title='Test',
-                          script=test_script(),
+                          script=test_script,
                           parameters=test_parameters,
                           namespace='os',
                           gui=True,
                           icon=['icon_1', 'icon_2'],
-                          setvars={'os.hashvar_0': {'value1': 'new_1',
-                                                    'value2': 'new_2'},
+                          setvars={'os.hashvar_0':
+                                   {'value1': 'new_1',
+                                    'value2': 'new_2'},
                                    '.linux.test_5': 1349},
                           rights=['install'])
 
-        command.initialize(datavars, IOModule())
-        command.script.run()
+        command_runner = command.make_runner(datavars, IOModule())
+        command_runner.run_command()
 
         assert datavars.os.hashvar_0.get_hash() == {'value1': 'new_1',
                                                     'value2': 'new_2'}

tests/variables/test_parameters.py → tests/parameters/test_parameters.py

@@ -2,29 +2,29 @@ import os
 import shutil
 import pytest
 from collections import OrderedDict
-from calculate.variables.parameters import BaseParameter, Integer, String,\
-                                           Description, Parameters,\
-                                           ValidationError, Choice, Bool,\
-                                           List, CyclicValidationError,\
-                                           ParameterError, Table, TableValue
+from calculate.parameters.parameters import BaseParameter, Integer, String,\
+                                            Description, Parameters,\
+                                            ValidationError, Choice, Bool,\
+                                            List, CyclicValidationError,\
+                                            ParameterError, Table, TableValue
 from calculate.variables.loader import Datavars
 
 
-TESTFILES_PATH = os.path.join(os.getcwd(), 'tests/variables/testfiles')
+TESTFILES_PATH = os.path.join(os.getcwd(), 'tests/parameters/testfiles')
+GENTOO_PATH = os.path.join(TESTFILES_PATH, 'gentoo')
 
 
 @pytest.mark.parameters
 class TestParameters:
     def test_to_make_testfiles(self):
         shutil.copytree(os.path.join(TESTFILES_PATH, 'gentoo.backup'),
-                        os.path.join(TESTFILES_PATH, 'gentoo'),
-                        symlinks=True)
+                        os.path.join(TESTFILES_PATH, 'gentoo'), symlinks=True)
 
     def test_if_some_parameters_are_created_using_classes_inheriting_the_BaseParameter_class_and_its_types_is_set_in_the_parameter_s_classes__the_parameters_can_be_created_using_this_classes_and_their_instances_types_are_types_from_their_classes(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
+                                                        'variables'))
 
-        PARAMS = Parameters().initialize(datavars)
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class MyShinyParameter(BaseParameter):
@@ -75,6 +75,17 @@ class TestParameters:
             OneMoreParameter('one-more', 'The Others',
                              Description(full='Plan 9 from outer space.')))
 
+        class AnotherParameter(BaseParameter):
+            type = String()
+
+            def bind_method(self):
+                return 'default string', None
+
+            def validate(self, value):
+                if not value.startswith('/var/lib'):
+                    raise ValidationError("The value must starts with a"
+                                          " '/var/lib'")
+
         shiny_one, another_one, weird_one = PARAMS
 
         shiny_one.set(9)
@@ -88,9 +99,9 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_bind_method_with_a_variable_in_its_arguments__the_default_parameter_value_is_calculated_using_this_method_and_a_variable_from_arguments_can_invalidate_the_parameter(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
+                                                        'variables'))
 
-        PARAMS = Parameters().initialize(datavars)
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class TestParameter(BaseParameter):
@@ -120,9 +131,9 @@ class TestParameters:
 
     def test_if_bind_method_is_set_for_parameter_and_then_the_set_method_is_used_to_change_value__the_parameters_value_is_changed_and_a_variable_from_a_bounded_variable_is_not_able_to_invalidate_parameter_s_value(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
+                                                        'variables'))
 
-        PARAMS = Parameters().initialize(datavars)
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class TestParameter(BaseParameter):
@@ -151,9 +162,9 @@ class TestParameters:
 
     def test_if_the_bind_method_is_set_for_two_variables__the_bind_method_calculates_parameter_s_default_value__variables_can_invalidate_parameter_s_value_before_the_set_method_is_used__the_set_parameter_can_change_value_of_the_parameter(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
+                                                        'variables'))
 
-        PARAMS = Parameters().initialize(datavars)
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class TestParameter(BaseParameter):
@@ -193,9 +204,9 @@ class TestParameters:
 
     def test_if_hash_value_is_in_the_set_variables_list__the_parameter_is_able_to_change_that_hash_s_value(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
+                                                        'variables'))
 
-        PARAMS = Parameters().initialize(datavars)
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class TestParameter(BaseParameter):
@@ -227,9 +238,9 @@ class TestParameters:
 
     def test_if_bind_method_is_set_for_parameter__the_method_can_return_as_second_value_of_the_return_tuple_disactivity_comment_witch_disactivates_the_parameter_even_if_parameters_values_is_set_by_user(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
+                                                        'variables'))
 
-        PARAMS = Parameters().initialize(datavars)
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class BoolTestParameter(BaseParameter):
@@ -282,8 +293,8 @@ class TestParameters:
 
     def test_Choice_type_is_set_for_parameter_and_bind_method_is_set_too__the_bind_method_can_define_available_values_for_the_choice_parameter__choice_availables_is_invalidatable_even_if_the_value_is_set_by_user__choice_type_checks_if_the_new_value_is_the_availables_list(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class TestParameter(BaseParameter):
@@ -312,8 +323,8 @@ class TestParameters:
 
     def test_if_some_parameters_are_created_with_some_types_and_there_are_attempts_to_assign_some_values_which_type_is_not_correct_to_this_parameters__the_parameters_types_checks_this_values_and_raises_ValidationError_exception(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -365,8 +376,8 @@ class TestParameters:
 
     def test_if_some_parameters_is_created_with_the_validate_methods__the_validate_methods_are_used_for_validating_all_values_that_is_set_to_the_parameter(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -456,8 +467,8 @@ class TestParameters:
 
     def test_if_some_parameters_are_created_with_validation_methods_that_use_parameters_forming_cyclic_validation_process__the_parameters_throw_the_CyclicValidationError_exception(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -540,8 +551,8 @@ class TestParameters:
 
     def test_if_two_parameters_is_created_with_the_bind_methods_and_one_of_them_can_be_disactivated_depending_on_the_variable_and_other_one_can_change_this_variable_s_value__the_second_parameter_can_disactivate_the_first_one_through_variable(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -586,8 +597,8 @@ class TestParameters:
 
     def test_if_there_is_an_attempt_to_add_in_the_parameters_container_new_parameter_but_the_container_already_has_parameter_with_the_same_fullname__the_parameters_container_throws_ParameterError_exception(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -617,8 +628,8 @@ class TestParameters:
 
     def test_if_there_is_an_attempt_to_add_in_the_parameters_container_new_parameter_but_the_container_already_has_parameter_with_the_same_shortname__the_parameters_container_throws_ParameterError_exception(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -650,8 +661,8 @@ class TestParameters:
 
     def test_if_some_parameters_are_created_with_different_argv_values__their_position_number_will_be_saved_in_the_parameters_container(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -695,8 +706,8 @@ class TestParameters:
 
     def test_if_there_is_an_attempt_to_add_in_the_parameters_container_new_parameter_but_the_container_already_has_parameter_with_the_same_argv_value__the_parameters_container_throws_ParameterError_exception(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -728,8 +739,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_the_bind_method__the_bind_method_can_get_access_to_the_current_parameter_value_during_the_value_calculation(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -773,8 +784,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_table_type_and_bind_method_is_set_for_this_parameter__the_bind_method_can_be_used_for_setting_of_the_table_fields_and_its_types(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -810,8 +821,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_table_type_and_types_of_its_fields_is_set__this_parameter_uses_this_types_for_invalidating_of_all_fields_values(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -870,8 +881,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_table_type_and_expandable_flag_is_True__new_rows_can_be_added_to_the_created_table_and_existing_rows_can_be_modified(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -914,8 +925,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_table_type_and_expandable_flag_is_False__existing_rows_can_be_modified_but_attempt_to_add_new_will_call_set_error_method_that_can_be_set_for_table(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -967,8 +978,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_table_type_and_to_set_method_is_used_to_add_variables_to_set_value__the_table_parameter_will_set_its_value_to_the_variable_from_set_list(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)
+                                                        'variables'))
+        PARAMS = Parameters(datavars)
 
         # Описание классов параметров.
         class FirstTestParameter(BaseParameter):
@@ -1010,8 +1021,8 @@ class TestParameters:
 
     def test_if_parameter_is_created_with_table_type_and_fill_method_is_set__the_parameter_will_use_fill_method_to_fill_empty_fields_of_the_table(self):
         datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
-                                                        'variables_17'))
-        PARAMS = Parameters().initialize(datavars)