Browse Source

Server development is started. Ability to run templates using scripts is added. fixed #14

master
Иванов Денис 9 months ago
parent
commit
778f665561
100 changed files with 1457 additions and 446 deletions
  1. +1
    -0
      .gitignore
  2. +141
    -63
      calculate/commands/commands.py
  3. +22
    -0
      calculate/logging.py
  4. +8
    -24
      calculate/parameters/parameters.py
  5. +194
    -56
      calculate/scripts/scripts.py
  6. +155
    -0
      calculate/server/server.py
  7. +38
    -0
      calculate/server/worker.py
  8. +0
    -9
      calculate/templates/template_engine.py
  9. +8
    -6
      calculate/templates/template_processor.py
  10. +36
    -29
      calculate/variables/loader.py
  11. +33
    -0
      client.py
  12. +2
    -1
      pytest.ini
  13. +6
    -0
      run_server.py
  14. +2
    -1
      run_templates.py
  15. +39
    -0
      tests/commands/parameters.py
  16. +14
    -0
      tests/commands/scripts.py
  17. +40
    -129
      tests/commands/test_commands.py
  18. +0
    -0
      tests/parameters/__init__.py
  19. +63
    -52
      tests/parameters/test_parameters.py
  20. +0
    -0
      tests/parameters/testfiles/gentoo.backup/ini_vars_0.backup
  21. +0
    -0
      tests/parameters/testfiles/gentoo.backup/portage/calculate.ini
  22. +0
    -0
      tests/parameters/testfiles/gentoo.backup/portage/profiles/calculate.ini
  23. +0
    -0
      tests/parameters/testfiles/gentoo.backup/portage/profiles/main/calculate.ini
  24. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/calculate.ini
  25. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/20/calculate.ini
  26. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/amd64/20/calculate.ini
  27. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/amd64/20/desktop/calculate.ini
  28. +3
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/amd64/20/desktop/parent
  29. +2
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/amd64/20/parent
  30. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/amd64/calculate.ini
  31. +1
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/amd64/parent
  32. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/calculate.ini
  33. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/desktop/calculate.ini
  34. +1
    -0
      tests/parameters/testfiles/gentoo.backup/repos/calculate/profiles/default/parent
  35. +2
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLD/amd64/calculate.ini
  36. +2
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLD/amd64/parent
  37. +21
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLD/calculate.ini
  38. +1
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLD/parent
  39. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLDX/amd64/calculate.ini
  40. +2
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLDX/amd64/parent
  41. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLDX/calculate.ini
  42. +1
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/CLDX/parent
  43. +0
    -0
      tests/parameters/testfiles/gentoo.backup/repos/distros/profiles/calculate.ini
  44. +0
    -0
      tests/parameters/testfiles/parameters_ini/calculate.ini
  45. +4
    -0
      tests/parameters/testfiles/variables/main/__init__.py
  46. +66
    -0
      tests/parameters/testfiles/variables/os/__init__.py
  47. +57
    -0
      tests/parameters/testfiles/variables/os/gentoo/__init__.py
  48. +19
    -0
      tests/parameters/testfiles/variables/system/__init__.py
  49. +142
    -76
      tests/scripts/test_scripts.py
  50. +0
    -0
      tests/scripts/testfiles/calculate.ini
  51. +2
    -0
      tests/scripts/testfiles/templates/action_1/.calculate_directory
  52. +1
    -0
      tests/scripts/testfiles/templates/action_1/dir_0/.calculate_directory
  53. +4
    -0
      tests/scripts/testfiles/templates/action_1/dir_0/file_0
  54. +2
    -0
      tests/scripts/testfiles/templates/action_2/.calculate_directory
  55. +1
    -0
      tests/scripts/testfiles/templates/action_2/dir_1/.calculate_directory
  56. +4
    -0
      tests/scripts/testfiles/templates/action_2/dir_1/file_0
  57. +0
    -0
      tests/scripts/testfiles/var.backup/db/pkg/test-category/other-package-1.1/CONTENTS
  58. +0
    -0
      tests/scripts/testfiles/var.backup/db/pkg/test-category/test-package-1.0/CONTENTS
  59. +1
    -0
      tests/scripts/testfiles/var.backup/db/pkg/test-category/test-package-1.0/SLOT
  60. BIN
      tests/scripts/testfiles/var.backup/lib/calculate/.config.swo
  61. +0
    -0
      tests/scripts/testfiles/var.backup/lib/calculate/config
  62. +3
    -0
      tests/scripts/testfiles/var.backup/lib/calculate/config-archive/etc/dir_6/file_0
  63. +3
    -0
      tests/scripts/testfiles/var.backup/lib/calculate/config-archive/etc/file_1
  64. +3
    -0
      tests/scripts/testfiles/var.backup/lib/calculate/config-archive/etc/file_12
  65. +3
    -0
      tests/scripts/testfiles/var.backup/lib/calculate/config-archive/etc/file_2
  66. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/ini_vars_0.backup
  67. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/portage/calculate.ini
  68. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/portage/profiles/calculate.ini
  69. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/portage/profiles/main/calculate.ini
  70. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/calculate.ini
  71. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/20/calculate.ini
  72. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/amd64/20/calculate.ini
  73. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/amd64/20/desktop/calculate.ini
  74. +3
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/amd64/20/desktop/parent
  75. +2
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/amd64/20/parent
  76. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/amd64/calculate.ini
  77. +1
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/amd64/parent
  78. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/calculate.ini
  79. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/desktop/calculate.ini
  80. +1
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/calculate/profiles/default/parent
  81. +2
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLD/amd64/calculate.ini
  82. +2
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLD/amd64/parent
  83. +21
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLD/calculate.ini
  84. +1
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLD/parent
  85. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLDX/amd64/calculate.ini
  86. +2
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLDX/amd64/parent
  87. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLDX/calculate.ini
  88. +1
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/CLDX/parent
  89. +0
    -0
      tests/scripts/testfiles/var.backup/lib/gentoo/repos/distros/profiles/calculate.ini
  90. +12
    -0
      tests/scripts/testfiles/variables/main/__init__.py
  91. +66
    -0
      tests/scripts/testfiles/variables/os/__init__.py
  92. +59
    -0
      tests/scripts/testfiles/variables/os/gentoo/__init__.py
  93. +19
    -0
      tests/scripts/testfiles/variables/system/__init__.py
  94. +58
    -0
      tests/server/test_server.py
  95. +0
    -0
      tests/server/testfiles/calculate.ini
  96. +54
    -0
      tests/server/testfiles/commands/cmd_module.py
  97. +0
    -0
      tests/server/testfiles/gentoo.backup/ini_vars_0.backup
  98. +0
    -0
      tests/server/testfiles/gentoo.backup/portage/calculate.ini
  99. +0
    -0
      tests/server/testfiles/gentoo.backup/portage/profiles/calculate.ini
  100. +0
    -0
      tests/server/testfiles/gentoo.backup/portage/profiles/main/calculate.ini

+ 1
- 0
.gitignore View File

@@ -1,6 +1,7 @@
*.pyc
*.pyo
*.swp
*.sock
build/
dist/
calculate_lib.egg-info/


+ 141
- 63
calculate/commands/commands.py View File

@@ -1,14 +1,13 @@
from types import FunctionType
from typing import Union, Callable, List
from calculate.scripts.scripts import Script
from calculate.variables.parameters import Parameters
from calculate.variables.datavars import NamespaceNode, DependenceAPI,\
DependenceError, VariableNotFoundError
from calculate.variables.loader import Datavars
from calculate.utils.io_module import IOModule
from ..scripts.scripts import Script
from ..parameters.parameters import BaseParameter, Parameters
from ..variables.datavars import NamespaceNode, DependenceAPI,\
VariableNotFoundError
from ..variables.loader import Datavars
from ..utils.io_module import IOModule


class CommandCreationError(Exception):
class CommandDescriptionError(Exception):
'''Исключение кидаемое при наличии ошибок во время создания объекта
команды.'''
def __init__(self, message: str, command_id: str = '', title: str = ''):
@@ -38,103 +37,182 @@ class CommandInitializationError(Exception):


class Command:
'''Класс команды, предназначен для хранения информации о команде и привязки ее к модулю ввода/вывода'''
def __init__(self, command_id: str = '', category: str = '',
'''Класс описания команды, предназначен для хранения информации о команде
и создания по данному описанию объекта лаунчера команды.'''
def __init__(self, command_id: str = '',
category: str = '',
title: str = '',
script: Union[Callable, Script, None] = None,
parameters: Union[Parameters, None] = None,
args: Union[tuple, list] = tuple(),
parameters: Union[List[BaseParameter], None] = None,
namespace: Union[str, NamespaceNode, None] = None,
command: str = '', gui: bool = False,
command: str = '',
gui: bool = False,
icon: Union[str, List[str]] = '',
setvars: dict = {}, rights: List[str] = []):
self._datavars: Union[Datavars, NamespaceNode, None] = None
setvars: dict = {},
rights: List[str] = []):
# Идентификатор команды обязателен.
if not command_id:
raise CommandCreationError('command ID is not set')
self._id: str = command_id
raise CommandDescriptionError('command ID is not set')
self.__id: str = command_id

# Если собственно команда не была указана, получаем ее из ID.
if command:
self._command: str = command
self.__command: str = command
else:
self._command: str = f'cl_{self._id}'
self.__command: str = f'cl_{self.__id}'

# Название команды.
if not title:
raise CommandCreationError("title is not set",
command_id=command_id)
self._title: str = title
raise CommandDescriptionError("title is not set",
command_id=command_id)
self.__title: str = title

# Категория, к которой относится команда.
if not category:
raise CommandCreationError("category is not set",
command_title=title)
self._category: str = category
raise CommandDescriptionError("category is not set",
command_title=title)
self.__category: str = category

# Пространство имен относительно которого, будет выполняться скрипт.
self._namespace: Union[str, NamespaceNode, None] = namespace
self.__namespace: Union[str, NamespaceNode, None] = namespace

# Параметры, указываемые при вызове этой команды.
if parameters is None:
raise CommandCreationError("parameters is not set",
command_title=title)
self._parameters: Parameters = parameters
raise CommandDescriptionError("parameters is not set",
command_title=title)
self.__parameters: List[BaseParameter] = parameters

# Скрипт выполняемый при вызове этой команды.
if not script:
raise CommandCreationError("script is not set",
command_title=title)
elif isinstance(script, FunctionType):
# Поддержка способа описания скрипта в функции.
self._script: Script = script()
else:
self._script: Script = script
raise CommandDescriptionError("script is not set",
command_title=title)
self.__script: Script = script
# Аргументы, с которыми будет запущен скрипт. Пока только статичные
# значения.
self.__args = args

# Параметр указывающий, должена ли данная команда отображаться в
# графическом интерфейсе.
self._gui: bool = gui
self.__gui: bool = gui

# Устанавливаем название иконки.
if not icon and self._gui:
raise CommandCreationError("icon is not set",
command_title=title)
self._icon: Union[str, List[str]] = icon
if not icon and self.__gui:
raise CommandDescriptionError("icon is not set",
command_title=title)
self.__icon: Union[str, List[str]] = icon

# Словарь с переменными и значениями, которые нужно им присвоить.
self._setvars = setvars
self.__setvars: dict = setvars

# Права, необходимые данной команде.
# TODO разобраться с тем, как использовать.
if not rights:
raise CommandCreationError("rights is not set",
title=title)
self._rights = rights
raise CommandDescriptionError("rights is not set",
title=title)
self.__rights = rights

def make_runner(self, datavars: Union[Datavars, NamespaceNode],
output: IOModule):
'''Метод создающий на основе данного описания команды экземпляр раннера.
'''
return CommandRunner(self, datavars, output)

# Интерфейс иммутабельного объекта описания команды.
@property
def id(self) -> str:
return self.__id

@property
def command(self) -> str:
return self.__command

@property
def title(self) -> str:
return self.__title

@property
def category(self) -> str:
return self.__category

@property
def namespace(self) -> str:
return self.__namespace

@property
def parameters(self) -> str:
return self.__parameters

@property
def script(self) -> Script:
return self._script
return self.__script

def initialize(self, datavars: Union[Datavars, NamespaceNode],
output: IOModule) -> 'Command':
'''Метод для инициализации всего, что нужно инициализировать в команде.
'''
@property
def args(self) -> tuple:
return self.__args

@property
def gui(self) -> bool:
return self.__gui

@property
def icon(self) -> Union[str, List[str]]:
return self.__icon

@property
def setvars(self) -> dict:
return self.__setvars

@property
def rights(self) -> List[str]:
return self.__rights

def __repr__(self):
return f"<Command: {self.__title}>"


class CommandRunner:
def __init__(self, command: Command,
datavars: Union[Datavars, NamespaceNode],
output: IOModule):
'''Класс инкапулирующий все данные о команде, а также необходимые для
нее параметры, модуль ввода-вывода, переменные и прочее. Предназначен
для конфигурации команды и запуска ее в воркере.'''
self._command = command
self._datavars = datavars
self._output = output

# Сначала найдем пространство имен.
if self._namespace is not None:
if isinstance(self._namespace, str):
self._namespace = self._find_namespace(self._namespace,
datavars)

# Ищем пространство имен.
if self._command.namespace is not None:
if isinstance(self._command.namespace, str):
self._namespace = self._find_namespace(
self._command.namespace,
datavars)
# Инициализируем параметры.
self._parameters.initialize(datavars)
self._parameters = Parameters(self._datavars)
self._parameters.add(*self._command.parameters)

# Получаем запускатель скрипта.
self._script_launcher = self._command.script.make_launcher(
output,
datavars,
self._namespace)
# Устанавливаем переменные, если нужно.
if self._command.setvars:
self._set_variables(self._command.setvars,
datavars, self._namespace)

# Инициализируем скрипт.
self._script.initialize(output, datavars, self._namespace)
def run_command(self):
args = self._command.args
self._script_launcher(*args)

# Устанавливаем переменные, если нужно.
if self._setvars:
self._set_variables(self._setvars, datavars, self._namespace)
def set_parameters(self, values: dict):
pass

return self
def get_parameters(self, group: Union[str, None] = None):
'''Метод для установки параметров.'''
pass

def _find_namespace(self, namespace_path: str,
datavars: Union[Datavars, NamespaceNode]


+ 22
- 0
calculate/logging.py View File

@@ -0,0 +1,22 @@
from logging import DEBUG

dictLogConfig = {
"version": 1,
"formatters": {
"form_1": {
"format":
'%(asctime)s %(levelname)s: %(message)s'
}
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "form_1",
"level": DEBUG
}
},
"root": {
"handlers": ['console'],
"level": DEBUG
}
}

calculate/variables/parameters.py → calculate/parameters/parameters.py View File

@@ -1,10 +1,10 @@
# vim: fileencoding=utf-8
#
from .datavars import DependenceAPI, VariableWrapper, NamespaceNode
from .loader import Datavars
from ..variables.datavars import DependenceAPI, VariableWrapper, NamespaceNode
from ..variables.loader import Datavars
from collections import OrderedDict
from contextlib import contextmanager
from typing import Tuple, Union
from typing import Tuple, Union, Any


class ParameterError(Exception):
@@ -541,7 +541,7 @@ class GroupWrapper:

class Parameters:
'''Класс контейнера параметров.'''
def __init__(self, datavars: Datavars = None, check_order=[]):
def __init__(self, datavars: Datavars, check_order=[]):
self._datavars = datavars
self._parameters = OrderedDict()
self._validation_dict = OrderedDict()
@@ -558,24 +558,8 @@ class Parameters:
# параметров.
self._gui_helpers = {}

self._initialized = False

def initialize(self, datavars: Union[Datavars, NamespaceNode]):
if not self._initialized:
self._datavars = datavars
if self._parameters:
parameters = self._parameters
self._parameters = OrderedDict()
self._add(parameters)
self._initialized = True
return self

def add(self, *parameters: Tuple[BaseParameter]):
'''Метод для добавления некоторой совокупности параметров.'''
if not self._initialized:
self._parameters = []
self._parameters.extend(parameters)

for parameter in parameters:
self.add_parameter(parameter)

@@ -671,12 +655,12 @@ class Parameters:
self._validation = False
self._validation_dict = OrderedDict()

def get_group_parameters(self, group_name):
def get_group_parameters(self, group_name: str):
'''Метод для получения списка параметров, относящихся к указанной
группе.'''
return self._parameters[group_name]

def get_descriptions(self):
def get_descriptions(self) -> dict:
'''Метод для получения словаря с описанием параметров.'''
output = OrderedDict()
for group, parameters in self._parameters.items():
@@ -700,10 +684,10 @@ class Parameters:
return output

@property
def datavars(self):
def datavars(self) -> Union[Datavars, NamespaceNode]:
return self._datavars

def __getitem__(self, name):
def __getitem__(self, name: str) -> Any:
for group, parameters in self._parameters.items():
for parameter in parameters:
if parameter._name == name or parameter._shortname == name:

+ 194
- 56
calculate/scripts/scripts.py View File

@@ -3,9 +3,10 @@
import re
import inspect
from typing import Callable, Any, Union, List, Generator
from calculate.templates.template_processor import DirectoryProcessor
from calculate.variables.datavars import DependenceAPI, DependenceError,\
VariableNode, NamespaceNode,\
HashType, TableType
HashType, TableType, StringType
from calculate.variables.loader import Datavars
from calculate.utils.io_module import IOModule
from collections.abc import Iterable, Mapping
@@ -35,79 +36,112 @@ class ConditionError(Exception):


class Script:
'''Класс скрипта, собирает задачи, обработчики, блоки и прочее. Принимает
аргументы.'''
def __init__(self, script_id: str, args: List[Any] = [],
'''Класс скрипта, собирает задачи, обработчики, блоки и прочее, хранит их
в себе. Создает экземпляры лаунчеров.'''
def __init__(self, script_id: str,
args: List[Any] = [],
success_message: str = None,
failed_message: str = None,
interrupt_message: str = None):
self._id: str = script_id
self._items: List[Union['Task', 'Block', 'Handler', 'Run']] = None
self.__items: List[Union['Task', 'Block', 'Handler', 'Run']] = None
self.__tasks_is_set: bool = False

self._args_names: list = args
self._args_vars: list = []
self.__args: list = args

self._success_message: str = success_message
self._failed_message: str = failed_message
self._interrupt_message: str = interrupt_message
self.__success_message: str = success_message
self.__failed_message: str = failed_message
self.__interrupt_message: str = interrupt_message

self._output: IOModule = None
self._datavars: Union[Datavars, NamespaceNode] = None
self._namespace: Union[NamespaceNode, None] = None
@property
def id(self) -> str:
return self._id

self._handlers: set = set()
self._initialized: bool = False
@property
def args(self) -> list:
return self.__args

@property
def items(self) -> list:
return self.__items

@property
def success_message(self) -> str:
return self.__success_message

@property
def failed_message(self) -> str:
return self.__failed_message

@property
def interrupt_message(self) -> str:
return self.__interrupt_message

def tasks(self, *items: List[Union['Task', 'Block', 'Handler', 'Run']]
) -> 'Script':
'''Метод для указания задач и контейнеров, из которых состоит скрипт.
'''
self._items = items
if self.__tasks_is_set:
raise ScriptError("Script object is immutable.")

self.__items = items
self.__tasks_is_set = bool(self.__items)
return self

def initialize(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode],
namespace: NamespaceNode = None) -> 'Script':
def make_launcher(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode],
namespace: NamespaceNode = None) -> 'ScriptLauncher':
'''Метод для создания экземпляра объекта, для запуска данного
скрипта.'''
return ScriptLauncher(self, output, datavars, namespace)


class ScriptLauncher:
def __init__(self, script: Script,
output: IOModule,
datavars: Union[Datavars, NamespaceNode],
namespace: NamespaceNode):
'''Метод для инициализации, состоящей в установке модулей вывода,
модуля переменных, текущего пространства имен, а также создании
переменных скрипта.'''
self._script = script

self._output = output
self._datavars = datavars
self._namespace = namespace

if not self._initialized:
self._script_namespace, self._args_vars =\
self.make_script_variables(self._id,
self._args_names,
self._datavars)
self._initialized = True
return self
self._script_namespace, self._args_vars = self.make_script_variables(
script.id,
script.args,
datavars)

def run(self) -> None:
'''Метод для запуска задач, содержащихся в данном скрипте.'''
essential_error = None
handlers_to_run = []
founded_handlers = []
handlers_to_run = set()

for task_item in self._items:
for task_item in self._script.items:
if isinstance(task_item, Handler):
if task_item.id in self._handlers:
handlers_to_run.append(task_item)
founded_handlers.append(task_item)
elif essential_error is None:
try:
output = task_item.run(self._output,
self._datavars,
self._script_namespace,
self._namespace)
self._handlers.update(output)
handlers_to_run.update(output)
except Exception as error:
if isinstance(error, TaskError):
self._handlers.update(error.handlers)
handlers_to_run.update(error.handlers)
essential_error = error

for handler in handlers_to_run:
for handler in founded_handlers:
if handler.id not in handlers_to_run:
continue
try:
task_item.run(self._output, self._datavars,
self._script_namespace, self._namespace)
handler.run(self._output, self._datavars,
self._script_namespace, self._namespace)
except Exception as error:
# TODO Разобраться что делать с ошибками в обработчике.
self._output.set_error(f"error in handler '{task_item.id}':"
@@ -123,7 +157,7 @@ class Script:
'''Метод для создания переменных скрипта. Создает пространство имен
скрипта и переменных аргументов.'''
if 'scripts' not in datavars:
scripts = Script.make_scripts_namespace(datavars)
scripts = ScriptLauncher.make_scripts_namespace(datavars)
else:
scripts = datavars.scripts

@@ -132,11 +166,14 @@ class Script:
scripts.add_namespace(current_script)
else:
current_script = scripts[script_id]
current_script.clear()
# current_script.clear()

args_vars = []
for arg in args:
args_vars.append(VariableNode(arg, current_script))
if arg not in current_script.variables:
args_vars.append(VariableNode(arg, current_script))
else:
args_vars.append(current_script.variables[arg])
return current_script, args_vars

@staticmethod
@@ -157,14 +194,15 @@ class Script:

def __call__(self, *args):
'''Метод для запуска скрипта с аргументами.'''
if len(args) < len(self._args_names):
raise ScriptError((f"script '{self._id}' missing"
f" {len(self._args_names) - len(args)} required"
" arguments: '") +
"', '".join(self._args_names[len(args):]) + "'")
elif len(args) > len(self._args_names):
raise ScriptError(f"script '{self._id}' takes"
f" {len(self._args_names)} arguments, but"
if len(args) < len(self._script.args):
raise ScriptError(
(f"script '{self._script.id}' missing"
f" {len(self._script.args) - len(args)} required"
" arguments: '") +
"', '".join(self._script.args[len(args):]) + "'")
elif len(args) > len(self._script.args):
raise ScriptError(f"script '{self._script.id}' takes"
f" {len(self._script.args)} arguments, but"
f" {len(args)} were given")

args_values = self._get_args_values(args, self._datavars,
@@ -191,10 +229,106 @@ class Script:
return args_values


class RunTemplate:
'''Класс запускателя наложения шаблонов.'''
def __init__(self, id: str = '',
action: str = '',
package: str = '',
chroot_path: str = '',
root_path: str = '',
dbpkg: bool = True,
essential: bool = True,
**group_packages):
self._id: str = id
self._action: str = action
self._package: str = package or None
self._chroot_path: str = chroot_path or None
self._root_path: str = root_path or None

self._dbpkg = dbpkg
self._essential = essential
self._groups: dict = group_packages

def _initialize(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode]):
# Проверяем наличие идентификатора.
if not self._id:
error_msg = "id is not set for templates runner"
if self._essential:
raise TaskError(error_msg)
else:
output.set_error(error_msg)
return set()

# Проверяем наличие параметра action, который обязателен.
if not self._action:
error_msg = ("action parameter is not set for templates runner"
f" '{self._id}'")
if self._essential:
raise TaskError(error_msg)
else:
output.set_error(error_msg)
return set()

# Если установлен chroot_path -- устанавливаем значение соответствующей
# переменной.
if self._chroot_path is not None:
if 'cl_chroot_path' in datavars.main:
datavars.main['cl_chroot_path'].source = self._chroot_path
else:
VariableNode("cl_chroot_path", datavars.main,
variable_type=StringType,
source=self._chroot_path)

# Если установлен root_path -- устанавливаем значение соответствующей
# переменной.
if self._root_path is not None:
if 'cl_root_path' in datavars.main:
datavars.main['cl_root_path'].source = self._root_path
else:
VariableNode("cl_root_path", datavars.main,
variable_type=StringType,
source=self._chroot_path)

if self._groups:
groups = list(self._groups.keys())
for group, atoms in groups:
if isinstance(atoms, str):
self._groups[group] = [name.strip() for name
in atoms.split(',')]

def run(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode],
script_namespace: NamespaceNode,
namespace: NamespaceNode) -> set:
'''Метод для запуска наложения шаблонов.'''
self._initialize(output, datavars)
template_processor = DirectoryProcessor(self._action,
datavars_module=datavars,
package=self._package,
output_module=output,
dbpkg=self._dbpkg,
**self._groups)
changed_files = template_processor.process_template_directories()
self._create_result_var(script_namespace,
changed_files=changed_files)
return set()

def _create_result_var(self, script_namespace: NamespaceNode,
changed_files: dict = {},
skipped: list = []) -> None:
'''Метод для создания переменной задачи, в которую отправляются
результаты вычислений, имеющихся в функции action.'''
VariableNode(self._id, script_namespace, variable_type=HashType,
source={"changed": changed_files,
"skipped": skipped})


class Run:
'''Класс запускателя скриптов.'''
def __init__(self, script: Script, namespace: str = None,
args: List[Any] = [], when: 'Var' = None,
args: List[Any] = [],
when: 'Var' = None,
essential: bool = True):
self._script: Script = script
self._namespace: str = namespace
@@ -204,9 +338,11 @@ class Run:

self._essential = essential

def run(self, output: IOModule, datavars: Union[Datavars, NamespaceNode],
parent_namespace: NamespaceNode, namespace: NamespaceNode) -> set:
'''Метод для запуска скрипта, указанного имеющегося в запускателе.'''
def run(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode],
parent_namespace: NamespaceNode,
namespace: NamespaceNode) -> set:
'''Метод для запуска скрипта, указанного в запускателе.'''
if self._condition is None or self._condition(datavars,
namespace,
parent_namespace):
@@ -222,8 +358,10 @@ class Run:

try:
# if not self._script._initialized:
self._script.initialize(output, datavars, self._namespace)
self._script(*self._args)
launcher = self._script.make_launcher(
output, datavars,
namespace=self._namespace)
launcher(*self._args)
except (ScriptError, TaskError) as error:
if self._essential:
raise ScriptError(f"essential script '{self._script._id}'"
@@ -330,8 +468,6 @@ class Task:
else:
self._args = self._update_arguments(self._args_sources)

return self

def run(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode],
script_namespace: NamespaceNode,
@@ -664,7 +800,8 @@ class Until(Loop):

class Block:
'''Класс блока задач.'''
def __init__(self, *tasks: List[Task], when: Union['Var', None] = None,
def __init__(self, *tasks: List[Task],
when: Union['Var', None] = None,
loop: Loop = Loop()):
self._tasks: List[Task] = tasks
self._rescue_tasks: List[Task] = None
@@ -736,7 +873,8 @@ class Handler:
def id(self) -> str:
return self._id

def run(self, output: IOModule, datavars: Union[Datavars, NamespaceNode],
def run(self, output: IOModule,
datavars: Union[Datavars, NamespaceNode],
script_namespace: NamespaceNode,
namespace: NamespaceNode) -> None:
'''Метод для запуска выполнения задач, содержащихся в обработчике.'''


+ 155
- 0
calculate/server/server.py View File

@@ -0,0 +1,155 @@
from ..variables.loader import Datavars
from ..commands.commands import Command
from ..logging import dictLogConfig
from logging.config import dictConfig
from logging import getLogger
from typing import Callable
from fastapi import FastAPI
from .worker import Worker
import importlib
import uvicorn
import asyncio
import os


# TODO
# 1. Разобраться с описанием команд как ресурсов и со всем, что от них зависит.
# 2. Разобраться с объектами воркеров. И способом их функционирования.


class Server:
def __init__(self, socket_path='./input.sock',
datavars_path: str = 'calculate/vars/',
commands_path: str = 'calculate/commands'):
self._app = FastAPI()
self._socket_path = socket_path
self._event_loop = asyncio.get_event_loop()

# Конфигурируем логгирование.
dictConfig(dictLogConfig)
self._logger = getLogger("main")
self.log_msg = {'DEBUG': self._logger.debug,
'INFO': self._logger.info,
'WARNING': self._logger.warning,
'ERROR': self._logger.error,
'CRITICAL': self._logger.critical}

self._datavars = Datavars(variables_path=datavars_path,
logger=self._logger)

# Словарь описаний команд.
self._commands = self._get_commands_list(commands_path)

# Словарь CID и экземпляров команд, передаваемых воркерам.
self._commands_instances = {}

# Словарь WID и экземпляров процессов-воркеров, передаваемых воркерам.
self._workers = {}

# Соответствие путей обработчикам запросов для HTTP-метода GET.
self._add_routes(self._app.get,
{"/": self._get_root,
"/commands": self._get_commands,
"/commands/{cid}": self._get_command,
"/workers/{wid}": self._get_worker})
self._add_routes(self._app.post,
{"/commands/{command_id}": self._post_command})

def _get_commands_list(self, commands_path: str) -> list:
'''Метод для получения совокупности описаний команд.'''
output = {}
package = ".".join(commands_path.split("/"))

for entry in os.scandir(commands_path):
if (not entry.name.endswith('.py')
or entry.name in {"commands.py", "__init__.py"}):
continue
module_name = entry.name[:-3]
try:
module = importlib.import_module("{}.{}".format(package,
module_name))
for obj in dir(module):
if type(module.__getattribute__(obj)) == Command:
command_object = module.__getattribute__(obj)
output[command_object.id] = command_object
except Exception:
continue
return output

async def _get_root(self) -> dict:
'''Обработчик корневых запросов.'''
return {'msg': 'root msg'}

async def _get_commands(self) -> dict:
'''Обработчик, отвечающий на запросы списка команд.'''
response = {}
for command_id, command_object in self._commands.items():
response.update({command_id: {"title": command_object.title,
"category": command_object.category,
"icon": command_object.icon,
"command": command_object.command}})
return response

async def _get_command(self, cid: int) -> dict:
'''Обработчик запросов списка команд.'''
if cid not in self._commands_instances:
# TODO добавить какую-то обработку ошибки.
pass
return {'id': cid,
'name': f'command_{cid}'}

async def _get_worker(self, wid: int):
'''Тестовый '''
self._make_worker(wid=wid)
worker = self._workers[wid]
worker.run(None)
await worker.send({"text": "INFO"})
data = await worker.get()
if data['type'] == 'log':
self.log_msg[data['level']](data['msg'])
return data

async def _post_command(self, command_id: str) -> int:
if command_id not in self._commands:
# TODO добавить какую-то обработку ошибки.
pass
return

def _add_routes(self, method: Callable, routes: dict) -> None:
'''Метод для добавления методов.'''
for path, handler in routes.items():
router = method(path)
router(handler)

def _make_worker(self, wid: int = None):
'''Метод для создания воркера для команды.'''
worker = Worker(self._event_loop)
if wid is None:
self._workers[wid] = worker
return wid
elif not self._workers:
self._workers[0] = worker
return 0
else:
wid = max(self._workers.keys()) + 1
self._workers[wid] = worker
return wid

def _make_command(self, command_id: str) -> int:
'''Метод для создания команды по ее описанию.'''
command_description = self._commands[command]

@property
def app(self):
return self._app

def run(self):
'''Метод для запуска сервера.'''
# Выгружаем список команд.
uvicorn.run(self._app,
uds=self._socket_path)


if __name__ == '__main__':
server = Server()
server.run()

+ 38
- 0
calculate/server/worker.py View File

@@ -0,0 +1,38 @@
from multiprocessing import Queue, Process
# from time import sleep


class Worker:
def __init__(self, loop):
self._output_queue = Queue()
self._in_queue = Queue()
self._event_loop = loop

async def send(self, data: dict):
self._in_queue.put(data)

async def get(self):
data = await self._event_loop.run_in_executor(None, self._get_output,
self._output_queue)
return data

@staticmethod
def _get_output(output_queue: Queue) -> dict:
return output_queue.get()

@staticmethod
def _main_loop(command, in_queue: Queue, out_queue: Queue):
data = in_queue.get()
print('\nworker for command:', command)
output = {"type": "log",
"level": "INFO",
"msg": f"recieved message {data['text']}"}
out_queue.put(output)

def run(self, command):
'''Метод для запуска процесса воркера с заданным '''
worker_process = Process(target=self._main_loop,
args=(command,
self._in_queue,
self._output_queue))
worker_process.start()

+ 0
- 9
calculate/templates/template_engine.py View File

@@ -429,8 +429,6 @@ class ParametersProcessor:
real_path = parameter_value

# Ставим True, чтобы потом проверить этот параметр в postparse
print(f'source value {parameter_value}')
print(f'source <- {real_path}')
if not os.path.exists(real_path):
return True

@@ -592,10 +590,6 @@ class ParametersProcessor:
else:
groups = self._parameters_container.group

print(f'PACKAGE: {parameter_value}')
print(f'PARSED: {package_atom}')
print(f'GROUPS: {groups}')
print(f'UNINSTALL: {self._groups.get("uninstall", None)}')
for group in groups:
if group == 'install':
try:
@@ -610,7 +604,6 @@ class ParametersProcessor:
self._parameters_container.remove_parameter('package')
return

print('check failed')
raise ConditionFailed(f"package '{parameter_value}'"
" does not match the template condition",
self.lineno)
@@ -623,8 +616,6 @@ class ParametersProcessor:
if package[parameter] is not None:
if (group_package[parameter] is None
or group_package[parameter] != package[parameter]):
print(f'{group_package[parameter]} !='
f' {package[parameter]}')
break
else:
if package['use_flags'] is not None:


+ 8
- 6
calculate/templates/template_processor.py View File

@@ -719,8 +719,6 @@ class TemplateExecutor:
self.executor_output = {'target_path': None,
'stdout': None,
'stderr': None}
print('TARGET PATH:', target_path)

if parameters.append == 'skip':
return self.executor_output

@@ -1762,7 +1760,8 @@ class DirectoryTree:
class DirectoryProcessor:
'''Класс обработчика директорий шаблонов.'''
def __init__(self, action: str, datavars_module=Variables(), package='',
output_module=IOModule(), dbpkg=True, **groups):
output_module=IOModule(), dbpkg=True,
namespace: NamespaceNode = None, **groups):
if isinstance(action, list):
self.action = action
else:
@@ -1781,7 +1780,6 @@ class DirectoryProcessor:
if 'cl_root_path' in datavars_module.main:
self.templates_root = join_paths(self.cl_chroot_path,
datavars_module.main.cl_root_path)
print('ROOT PATH:', self.templates_root)
else:
self.templates_root = self.cl_chroot_path

@@ -1826,7 +1824,11 @@ class DirectoryProcessor:
# Разбираем atom имена пакетов, указанных для групп пакетов.
if groups:
for group_name, package_name in groups.items():
self._add_package_to_group(group_name, package_name)
if isinstance(package_name, list):
for atom in package_name:
self._add_package_to_group(group_name, atom)
else:
self._add_package_to_group(group_name, package_name)

# Инициализируем шаблонизатор.
self.template_engine = TemplateEngine(
@@ -1998,6 +2000,7 @@ class DirectoryProcessor:
self._run_exec_files()

self.template_executor.save_changes()
return self.template_executor.changed_files

def _execute_handlers(self):
'''Метод для запуска обработчиков добавленных в очередь обработчиков
@@ -2159,7 +2162,6 @@ class DirectoryProcessor:
необходимости, заполнения деревьев директорий шаблонов, с помощью
которых далее выполняются шаблоны пакетов из merge.'''
directory_name = os.path.basename(current_directory_path)
print('CURRENT TARGET PATH =', current_target_path)

# Если включено заполнение дерева создаем пустой словарь для сбора
# содержимого текущей директории.


+ 36
- 29
calculate/variables/loader.py View File

@@ -1,6 +1,7 @@
# vim: fileencoding=utf-8
#
import os
import logging
import importlib
import importlib.util
from jinja2 import Environment, FileSystemLoader
@@ -12,7 +13,6 @@ from calculate.variables.datavars import NamespaceNode, VariableNode,\
from calculate.utils.gentoo import ProfileWalker
from calculate.utils.files import read_file, FilesError
from calculate.utils.tools import Singleton
from calculate.utils.io_module import IOModule
from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
empty, printables, OneOrMore, lineno, line, SkipTo,\
LineEnd, Combine, nums
@@ -489,7 +489,7 @@ class VariableLoader:

def __init__(self, datavars, variables_path, repository_map=None):
self.datavars = datavars
self.output = datavars.output
self.logger = datavars.logger
self.ini_filler = NamespaceIniFiller()

self.variables_path = variables_path
@@ -519,9 +519,9 @@ class VariableLoader:
if section in current_namespace:
current_namespace = current_namespace[section]
else:
self.output.set_error("Variable 'os.gentoo.repositories'"
" is not found. Can not load profile"
" variables.")
self.logger.error("Variable 'os.gentoo.repositories'"
" is not found. Can not load profile"
" variables.")
return
self.repository_map = self._get_repository_map(self.datavars)

@@ -530,12 +530,12 @@ class VariableLoader:
'path' in self.datavars.os.gentoo.profile):
profile_path = self.datavars.os.gentoo.profile.path
else:
self.output.set_error("Variable 'os.gentoo.profile.path'"
" is not found. Can not load profile"
" variables.")
self.logger.error("Variable 'os.gentoo.profile.path'"
" is not found. Can not load profile"
" variables.")
return

self.output.set_info("Load variables from profile: '{}'.".format(
self.logger.info("Load variables from profile: '{}'.".format(
profile_path))
self._fill_from_profile_ini(profile_path)

@@ -546,20 +546,20 @@ class VariableLoader:
env_order = self.datavars.system.env_order
env_path = self.datavars.system.env_path
except VariableNotFoundError as error:
self.output.set_warning("Can not load additional variables: {}".
format(str(error)))
self.logger.warning("Can not load additional variables: {}".
format(str(error)))
return

for ini_file in env_order:
self.output.set_info("Loading variables from file: '{}'".format(
self.logger.info("Loading variables from file: '{}'".format(
ini_file))
if ini_file in env_path:
self.fill_from_custom_ini(env_path[ini_file].value)
self.output.set_success("Variables from '{}' are loaded".
format(ini_file))
self.logger.info("Variables from '{}' are loaded".format(
ini_file))
else:
self.output.set_warning("File '{}' is not found. Variables are"
" not loaded".format(ini_file))
self.logger.warning("File '{}' is not found. Variables are"
" not loaded".format(ini_file))

def _fill_from_package(self, current_namespace: NamespaceNode,
directory_path: str, package: str) -> None:
@@ -575,6 +575,8 @@ class VariableLoader:

# Сначала загружаем переменные из файлов.
for file_node in file_nodes:
if not file_node.name.endswith('.py'):
continue
file_name = file_node.name[:-3]
Namespace.set_current_namespace(current_namespace)
# with self.test(file_name, current_namespace):
@@ -603,8 +605,8 @@ class VariableLoader:
ini_file_text = read_file(file_path)
self.ini_filler.fill(self.datavars, ini_file_text)
except FilesError:
self.output.set_error("Can not load profile variables from"
" unexisting file: {}".format(file_path))
self.logger.error("Can not load profile variables from"
" unexisting file: {}".format(file_path))

def _get_repository_map(self, datavars):
'''Метод для получения из переменной словаря с репозиториями и путями
@@ -621,13 +623,13 @@ class VariableLoader:
parsing_errors = self.ini_filler.errors
if parsing_errors:
for error in parsing_errors:
self.output.set_error(error)
self.output.set_warning('Some variables was not loaded.')
self.logger.error(error)
self.logger.warning('Some variables was not loaded.')
else:
self.output.set_success('All variables are loaded.')
self.logger.info('All variables are loaded.')
else:
self.output.set_error("Variables are not loaded. File '{}' does"
" not exist.".format(file_path))
self.logger.error("Variables are not loaded. File '{}' does"
" not exist.".format(file_path))

@contextmanager
def test(self, file_name, namespace):
@@ -704,10 +706,16 @@ class CalculateIniSaver:
class Datavars:
'''Класс для хранения переменных и управления ими.'''
def __init__(self, variables_path='calculate/vars', repository_map=None,
io_module=IOModule()):
logger=None):
self._variables_path = variables_path
self._available_packages = self._get_available_packages()
self.output = io_module
if logger is not None:
self.logger = logger
else:
logger = logging.getLogger("main")
# stream_handler = logging.StreamHandler()
# logger.addHandler(stream_handler)
self.logger = logger

self.root = NamespaceNode('<root>')
self._loader = VariableLoader(self, self._variables_path,
@@ -749,8 +757,7 @@ class Datavars:

def _load_package(self, package_name):
'''Метод для загрузки переменных содержащихся в указанном пакете.'''
self.output.set_info("Loading datavars package '{}'".format(
package_name))
self.logger.info("Loading datavars package '{}'".format(package_name))
try:
self._loader.load_variables_package(package_name)
except Exception as error:
@@ -843,5 +850,5 @@ class Datavars:
dict_to_save = self.variables_to_save[target]
target_path = target_paths[target].value
saver.save_to_ini(target_path, dict_to_save)
self.output.set_info("Variables for '{}' is saved in the"
" file: {}".format(target, target_path))
self.logger.info("Variables for '{}' is saved in the"
" file: {}".format(target, target_path))

+ 33
- 0
client.py View File

@@ -0,0 +1,33 @@
import asyncio
import aiohttp


async def get_root(client):
async with client.get('http://localhost/') as resp:
assert resp.status == 200
return await resp.json()


async def main():
unix_conn = aiohttp.UnixConnector(path='./input.sock')

try:
async with aiohttp.ClientSession(connector=unix_conn) as client:
print('---------------------------')
print('Request GET "/" HTTP/1.1:')
response = await get_root(client)
print(f'Response: {response}')
print('---------------------------')
except KeyboardInterrupt:
raise
finally:
print('\nClosing connection...')
await unix_conn.close()


if __name__ == '__main__':
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(main())
except KeyboardInterrupt:
loop.close()

+ 2
- 1
pytest.ini View File

@@ -36,5 +36,6 @@ markers =
vars: marker for testing of the datavars module.
parameters: marker for testing of the parameters module.

tasks: marker for testing of the tasks.
scripts: marker for testing of the scripts.
commands: marker for testing of the commands.
server: marker for testing of the server.

+ 6
- 0
run_server.py View File

@@ -0,0 +1,6 @@
from calculate.server.server import Server


if __name__ == "__main__":
server = Server()
server.run()

+ 2
- 1
run_templates.py View File

@@ -40,7 +40,8 @@ def main():
datavars_module=datavars,
package=package,
output_module=io_module,
dbpkg=args.dbpkg)
dbpkg=args.dbpkg,
**group_packages)
template_processor.process_template_directories()




+ 39
- 0
tests/commands/parameters.py View File

@@ -0,0 +1,39 @@
from calculate.parameters.parameters import BaseParameter, Integer,\
String, ValidationError


class MyShinyParameter(BaseParameter):
type = Integer()

def validate(self, var):
if var.value < 10:
raise ValidationError("The value must be greater than 10")

def bind_method(self, var):
return var.value, None


class AnotherParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
if not value.startswith('/var/lib'):
raise ValidationError("The value must starts with a"
" '/var/lib'")


class OneMoreParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
available_set = {'mystery', 'horror', 'weird'}
if value not in available_set:
raise ValidationError(f"The value '{value}' is not in"
f" available. Available values:"
f" {', '.join(available_set)}")

+ 14
- 0
tests/commands/scripts.py View File

@@ -0,0 +1,14 @@
from calculate.scripts.scripts import Script, Task


def action(output, arg1: str) -> str:
'''Задача выполняемая скриптом ниже.'''
return f'os.calculate = {arg1}'


# Тестовый скрипт.
test_script = Script('test_script'
).tasks(Task(id='task',
name="Task",
action=action,
args=["os.calculate"]))

+ 40
- 129
tests/commands/test_commands.py View File

@@ -1,14 +1,15 @@
import os
import shutil
import pytest

from calculate.commands.commands import Command
from calculate.scripts.scripts import Script, Task
from calculate.parameters.parameters import Description
from calculate.variables.loader import Datavars
from calculate.variables.parameters import Parameters, BaseParameter, Integer,\
String, ValidationError,\
Description
from calculate.utils.io_module import IOModule

from .parameters import MyShinyParameter, OneMoreParameter, AnotherParameter
from .scripts import test_script


TESTFILES_PATH = os.path.join(os.getcwd(), 'tests/commands/testfiles')

@@ -24,78 +25,37 @@ class TestCommands:
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables'))

# Скрипт выполняемый командой.
def action(output: IOModule, arg1: str) -> str:
return f'os.calculate = {arg1}'

# Для обеспечения многопоточности вероятно придется делать так.
def test_script():
return Script('test_script'
).tasks(Task(id='task',
name="Task",
action=action,
args=["os.calculate"])
)

# Параметры данной команды.
test_parameters = Parameters().initialize(datavars)

class MyShinyParameter(BaseParameter):
type = Integer()

def validate(self, var):
if var.value < 10:
raise ValidationError("The value must be greater than 10")

def bind_method(self, var):
return var.value, None

class AnotherParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
if not value.startswith('/var/lib'):
raise ValidationError("The value must starts with a"
" '/var/lib'")

class OneMoreParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
available_set = {'mystery', 'horror', 'weird'}
if value not in available_set:
raise ValidationError(f"The value '{value}' is not in"
f" available. Available values:"
f" {', '.join(available_set)}")

test_parameters.add(
MyShinyParameter('my-shiny', 'The Shiniest ones',
Description(
short='The shiny parameter',
full='The shiniest thing known to science.',
usage='-S COUNT, --my-shiny COUNT'),
shortname='S').bind('os.linux.test_5'),
AnotherParameter('my-other', 'The Others',
Description(short='Not from this world.'),
shortname='o'),
OneMoreParameter('one-more', 'The Others',
Description(full='Plan 9 from outer space.')))

command = Command(command_id='test',
category='Test Category',
title='Test',
script=test_script(),
parameters=test_parameters,
script=test_script,
namespace='os',
rights=['group'])
command.initialize(datavars, IOModule())
command.script.run()
rights=['group'],
parameters=[
MyShinyParameter
(
'my-shiny', 'The Shiniest ones',
Description(
short='The shiny parameter',
full='The shiniest thing known to science.',
usage='-S COUNT, --my-shiny COUNT'),
shortname='S'
).bind('os.linux.test_5'),
AnotherParameter
(
'my-other', 'The Others',
Description(
short='Not from this world.'),
shortname='o'
),
OneMoreParameter
(
'one-more', 'The Others',
Description(full='Plan 9 from outer space.')
)])

command_runner = command.make_runner(datavars, IOModule())
command_runner.run_command()

assert datavars.scripts.test_script.task.result ==\
'os.calculate = test1 test2'
@@ -104,57 +64,7 @@ class TestCommands:
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables'))

# Скрипт выполняемый командой.
def action(output: IOModule, arg1: str) -> str:
return f'os.calculate = {arg1}'

# Для обеспечения многопоточности вероятно придется делать так.
def test_script():
return Script('test_script'
).tasks(Task(id='task',
name="Task",
action=action,
args=["os.calculate"])
)

# Параметры данной команды.
test_parameters = Parameters().initialize(datavars)

class MyShinyParameter(BaseParameter):
type = Integer()

def validate(self, var):
if var.value < 10:
raise ValidationError("The value must be greater than 10")

def bind_method(self, var):
return var.value, None

class AnotherParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
if not value.startswith('/var/lib'):
raise ValidationError("The value must starts with a"
" '/var/lib'")

class OneMoreParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
available_set = {'mystery', 'horror', 'weird'}
if value not in available_set:
raise ValidationError(f"The value '{value}' is not in"
f" available. Available values:"
f" {', '.join(available_set)}")

test_parameters.add(
test_parameters = [
MyShinyParameter('my-shiny', 'The Shiniest ones',
Description(
short='The shiny parameter',
@@ -165,23 +75,24 @@ class TestCommands:
Description(short='Not from this world.'),
shortname='o'),
OneMoreParameter('one-more', 'The Others',
Description(full='Plan 9 from outer space.')))
Description(full='Plan 9 from outer space.'))]

command = Command(command_id='test',
category='Test Category',
title='Test',
script=test_script(),
script=test_script,
parameters=test_parameters,
namespace='os',
gui=True,
icon=['icon_1', 'icon_2'],
setvars={'os.hashvar_0': {'value1': 'new_1',
'value2': 'new_2'},
setvars={'os.hashvar_0':
{'value1': 'new_1',
'value2': 'new_2'},
'.linux.test_5': 1349},
rights=['install'])

command.initialize(datavars, IOModule())
command.script.run()
command_runner = command.make_runner(datavars, IOModule())
command_runner.run_command()

assert datavars.os.hashvar_0.get_hash() == {'value1': 'new_1',
'value2': 'new_2'}


+ 0
- 0
tests/parameters/__init__.py View File


tests/variables/test_parameters.py → tests/parameters/test_parameters.py View File

@@ -2,29 +2,29 @@ import os
import shutil
import pytest
from collections import OrderedDict
from calculate.variables.parameters import BaseParameter, Integer, String,\
Description, Parameters,\
ValidationError, Choice, Bool,\
List, CyclicValidationError,\
ParameterError, Table, TableValue
from calculate.parameters.parameters import BaseParameter, Integer, String,\
Description, Parameters,\
ValidationError, Choice, Bool,\
List, CyclicValidationError,\
ParameterError, Table, TableValue
from calculate.variables.loader import Datavars


TESTFILES_PATH = os.path.join(os.getcwd(), 'tests/variables/testfiles')
TESTFILES_PATH = os.path.join(os.getcwd(), 'tests/parameters/testfiles')
GENTOO_PATH = os.path.join(TESTFILES_PATH, 'gentoo')


@pytest.mark.parameters
class TestParameters:
def test_to_make_testfiles(self):
shutil.copytree(os.path.join(TESTFILES_PATH, 'gentoo.backup'),
os.path.join(TESTFILES_PATH, 'gentoo'),
symlinks=True)
os.path.join(TESTFILES_PATH, 'gentoo'), symlinks=True)

def test_if_some_parameters_are_created_using_classes_inheriting_the_BaseParameter_class_and_its_types_is_set_in_the_parameter_s_classes__the_parameters_can_be_created_using_this_classes_and_their_instances_types_are_types_from_their_classes(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
'variables'))

PARAMS = Parameters().initialize(datavars)
PARAMS = Parameters(datavars)

# Описание классов параметров.
class MyShinyParameter(BaseParameter):
@@ -75,6 +75,17 @@ class TestParameters:
OneMoreParameter('one-more', 'The Others',
Description(full='Plan 9 from outer space.')))

class AnotherParameter(BaseParameter):
type = String()

def bind_method(self):
return 'default string', None

def validate(self, value):
if not value.startswith('/var/lib'):
raise ValidationError("The value must starts with a"
" '/var/lib'")

shiny_one, another_one, weird_one = PARAMS

shiny_one.set(9)
@@ -88,9 +99,9 @@ class TestParameters:

def test_if_parameter_is_created_with_bind_method_with_a_variable_in_its_arguments__the_default_parameter_value_is_calculated_using_this_method_and_a_variable_from_arguments_can_invalidate_the_parameter(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
'variables'))

PARAMS = Parameters().initialize(datavars)
PARAMS = Parameters(datavars)

# Описание классов параметров.
class TestParameter(BaseParameter):
@@ -120,9 +131,9 @@ class TestParameters:

def test_if_bind_method_is_set_for_parameter_and_then_the_set_method_is_used_to_change_value__the_parameters_value_is_changed_and_a_variable_from_a_bounded_variable_is_not_able_to_invalidate_parameter_s_value(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
'variables'))

PARAMS = Parameters().initialize(datavars)
PARAMS = Parameters(datavars)

# Описание классов параметров.
class TestParameter(BaseParameter):
@@ -151,9 +162,9 @@ class TestParameters:

def test_if_the_bind_method_is_set_for_two_variables__the_bind_method_calculates_parameter_s_default_value__variables_can_invalidate_parameter_s_value_before_the_set_method_is_used__the_set_parameter_can_change_value_of_the_parameter(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
'variables'))

PARAMS = Parameters().initialize(datavars)
PARAMS = Parameters(datavars)

# Описание классов параметров.
class TestParameter(BaseParameter):
@@ -193,9 +204,9 @@ class TestParameters:

def test_if_hash_value_is_in_the_set_variables_list__the_parameter_is_able_to_change_that_hash_s_value(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
'variables'))

PARAMS = Parameters().initialize(datavars)
PARAMS = Parameters(datavars)

# Описание классов параметров.
class TestParameter(BaseParameter):
@@ -227,9 +238,9 @@ class TestParameters:

def test_if_bind_method_is_set_for_parameter__the_method_can_return_as_second_value_of_the_return_tuple_disactivity_comment_witch_disactivates_the_parameter_even_if_parameters_values_is_set_by_user(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
'variables'))

PARAMS = Parameters().initialize(datavars)
PARAMS = Parameters(datavars)

# Описание классов параметров.
class BoolTestParameter(BaseParameter):
@@ -282,8 +293,8 @@ class TestParameters:

def test_Choice_type_is_set_for_parameter_and_bind_method_is_set_too__the_bind_method_can_define_available_values_for_the_choice_parameter__choice_availables_is_invalidatable_even_if_the_value_is_set_by_user__choice_type_checks_if_the_new_value_is_the_availables_list(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class TestParameter(BaseParameter):
@@ -312,8 +323,8 @@ class TestParameters:

def test_if_some_parameters_are_created_with_some_types_and_there_are_attempts_to_assign_some_values_which_type_is_not_correct_to_this_parameters__the_parameters_types_checks_this_values_and_raises_ValidationError_exception(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -365,8 +376,8 @@ class TestParameters:

def test_if_some_parameters_is_created_with_the_validate_methods__the_validate_methods_are_used_for_validating_all_values_that_is_set_to_the_parameter(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -456,8 +467,8 @@ class TestParameters:

def test_if_some_parameters_are_created_with_validation_methods_that_use_parameters_forming_cyclic_validation_process__the_parameters_throw_the_CyclicValidationError_exception(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -540,8 +551,8 @@ class TestParameters:

def test_if_two_parameters_is_created_with_the_bind_methods_and_one_of_them_can_be_disactivated_depending_on_the_variable_and_other_one_can_change_this_variable_s_value__the_second_parameter_can_disactivate_the_first_one_through_variable(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -586,8 +597,8 @@ class TestParameters:

def test_if_there_is_an_attempt_to_add_in_the_parameters_container_new_parameter_but_the_container_already_has_parameter_with_the_same_fullname__the_parameters_container_throws_ParameterError_exception(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -617,8 +628,8 @@ class TestParameters:

def test_if_there_is_an_attempt_to_add_in_the_parameters_container_new_parameter_but_the_container_already_has_parameter_with_the_same_shortname__the_parameters_container_throws_ParameterError_exception(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -650,8 +661,8 @@ class TestParameters:

def test_if_some_parameters_are_created_with_different_argv_values__their_position_number_will_be_saved_in_the_parameters_container(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -695,8 +706,8 @@ class TestParameters:

def test_if_there_is_an_attempt_to_add_in_the_parameters_container_new_parameter_but_the_container_already_has_parameter_with_the_same_argv_value__the_parameters_container_throws_ParameterError_exception(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -728,8 +739,8 @@ class TestParameters:

def test_if_parameter_is_created_with_the_bind_method__the_bind_method_can_get_access_to_the_current_parameter_value_during_the_value_calculation(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -773,8 +784,8 @@ class TestParameters:

def test_if_parameter_is_created_with_table_type_and_bind_method_is_set_for_this_parameter__the_bind_method_can_be_used_for_setting_of_the_table_fields_and_its_types(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -810,8 +821,8 @@ class TestParameters:

def test_if_parameter_is_created_with_table_type_and_types_of_its_fields_is_set__this_parameter_uses_this_types_for_invalidating_of_all_fields_values(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -870,8 +881,8 @@ class TestParameters:

def test_if_parameter_is_created_with_table_type_and_expandable_flag_is_True__new_rows_can_be_added_to_the_created_table_and_existing_rows_can_be_modified(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -914,8 +925,8 @@ class TestParameters:

def test_if_parameter_is_created_with_table_type_and_expandable_flag_is_False__existing_rows_can_be_modified_but_attempt_to_add_new_will_call_set_error_method_that_can_be_set_for_table(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -967,8 +978,8 @@ class TestParameters:

def test_if_parameter_is_created_with_table_type_and_to_set_method_is_used_to_add_variables_to_set_value__the_table_parameter_will_set_its_value_to_the_variable_from_set_list(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)
'variables'))
PARAMS = Parameters(datavars)

# Описание классов параметров.
class FirstTestParameter(BaseParameter):
@@ -1010,8 +1021,8 @@ class TestParameters:

def test_if_parameter_is_created_with_table_type_and_fill_method_is_set__the_parameter_will_use_fill_method_to_fill_empty_fields_of_the_table(self):
datavars = Datavars(variables_path=os.path.join(TESTFILES_PATH,
'variables_17'))
PARAMS = Parameters().initialize(datavars)