Server is partly implemented.

calculate/parameters/parameters.py

@@ -8,11 +8,11 @@ from typing import Tuple, Union, Any
 
 
 class ParameterError(Exception):
-    pass
+    ...
 
 
 class ValidationError(ParameterError):
-    pass
+    ...
 
 
 class CyclicValidationError(ValidationError):
@@ -372,7 +372,7 @@ class BaseParameter:
     def validate(self, container, datavars, value) -> None:
         '''Метод для проверки корректности параметра. Переопределяется при
         создании нового типа параметра.'''
-        pass
+        return None
 
     def validate_value(self, value):
         '''Метод для запуска валидации параметра с использованием

calculate/scripts/scripts.py

@@ -4,9 +4,15 @@ import re
 import inspect
 from typing import Callable, Any, Union, List, Generator
 from calculate.templates.template_processor import DirectoryProcessor
-from calculate.variables.datavars import DependenceAPI, DependenceError,\
+from calculate.variables.datavars import (
-                                         VariableNode, NamespaceNode,\
+    DependenceAPI,
-                                         HashType, TableType, StringType
+    DependenceError,
+    VariableNode,
+    NamespaceNode,
+    HashType,
+    TableType,
+    StringType,
+    )
 from calculate.variables.loader import Datavars
 from calculate.utils.io_module import IOModule
 from collections.abc import Iterable, Mapping

calculate/server/server.py

@@ -76,6 +76,7 @@ class Server:
                 continue
         return output
 
+    # Обработчики запросов серверу.
     async def _get_root(self) -> dict:
         '''Обработчик корневых запросов.'''
         return {'msg': 'root msg'}
@@ -99,7 +100,7 @@ class Server:
                 'name': f'command_{cid}'}
 
     async def _get_worker(self, wid: int):
-        '''Тестовый '''
+        '''Тестовый обработчик.'''
         self._make_worker(wid=wid)
         worker = self._workers[wid]
         worker.run(None)
@@ -115,6 +116,9 @@ class Server:
             pass
         return
 
+    # Обработчики сообщений воркеров.
+
+    # Вспомогательные методы.
     def _add_routes(self, method: Callable, routes: dict) -> None:
         '''Метод для добавления методов.'''
         for path, handler in routes.items():
@@ -123,16 +127,15 @@ class Server:
 
     def _make_worker(self, wid: int = None):
         '''Метод для создания воркера для команды.'''
-        worker = Worker(self._event_loop)
+        if wid is not None:
-        if wid is None:
+            self._workers[wid] = Worker(wid, self._event_loop)
-            self._workers[wid] = worker
             return wid
         elif not self._workers:
-            self._workers[0] = worker
+            self._workers[0] = Worker(0, self._event_loop)
             return 0
         else:
             wid = max(self._workers.keys()) + 1
-            self._workers[wid] = worker
+            self._workers[wid] = Worker(wid, self._event_loop)
             return wid
 
     def _make_command(self, command_id: str) -> int:

calculate/server/worker.py

@@ -1,21 +1,173 @@
+import os
+import json
+import socket
+import logging
+from typing import Union
+from ..variables.loader import Datavars
 from multiprocessing import Queue, Process
+from ..commands.commands import CommandRunner, Command
 # from time import sleep
 
 
-class Worker:
+class WorkerIOError(KeyboardInterrupt):
-    def __init__(self, loop):
+    pass
-        self._output_queue = Queue()
+
-        self._in_queue = Queue()
+
-        self._event_loop = loop
+class IOModule:
+    '''Класс модуля ввода/вывода для воркеров.'''
+    def __init__(self, socket_path: str):
+        self._sock = socket.socket(socket.AF_UNIX,
+                                   socket.SOCK_STREAM)
+        self._sock.bind(socket_path)
+        self._sock.listen()
+        self._connection = None
+
+    def input(self, msg: str) -> str:
+        '''Метод через который возможен ввод данных в скрипт.'''
+        input_request = json.dumps({"type": "input",
+                                    "msg": msg}) + '\0'
+        answer = None
+        while answer is None:
+            if not self._check_connection(self._connection):
+                self._connection = self._make_connection()
+            self._connection.sendall(input_request.encode())
+            answer = self._get()
+        return answer['data']
+
+    def set_debug(self, msg: str) -> None:
+        self.output(msg, level=logging.DEBUG)
+
+    def set_info(self, msg: str) -> None:
+        self.output(msg, level=logging.INFO)
 
-    async def send(self, data: dict):
+    def set_warning(self, msg: str) -> None:
-        self._in_queue.put(data)
+        self.output(msg, level=logging.WARNING)
 
-    async def get(self):
+    def set_error(self, msg: str) -> None:
-        data = await self._event_loop.run_in_executor(None, self._get_output,
+        self.output(msg, level=logging.ERROR)
-                                                      self._output_queue)
+
+    def set_critical(self, msg: str) -> None:
+        self.output(msg, level=logging.CRITICAL)
+
+    def output(self, msg: str, level: int = logging.INFO) -> None:
+        '''Метод для отправки серверу вывода с указанием уровня.'''
+        if not self._check_connection(self._connection):
+            self._connection = self._make_connection()
+        output_request = json.dumps({"type": "output",
+                                     "level": level,
+                                     "msg": msg}) + '\0'
+        self._connection.sendall(output_request.encode())
+
+    def send(self, data: dict) -> None:
+        '''Метод для отправки данных серверу.'''
+        if not self._check_connection(self._connection):
+            self._connection = self._make_connection()
+        data = json.dumps(data) + '\0'
+        self._connection.sendall(data.encode())
+
+    def receive(self) -> dict:
+        '''Метод для получения данных от сервера.'''
+        data = None
+        while data is None:
+            if not self._check_connection(self._connection):
+                self._connection = self._make_connection()
+            data = self._get()
         return data
 
+    def _get(self) -> Union[None, dict]:
+        '''Метод для считывания данных, возможно, поделенных на кадры.'''
+        try:
+            data = b''
+            while True:
+                chunk = self._connection.recv(1024)
+                if not chunk:
+                    return None
+
+                data += chunk
+                if not data.endswith(b'\0'):
+                    if b'\0' in data:
+                        # Если после символа конца сообщения есть еще какие-то
+                        # данные -- считаем их наличие ошибкой.
+                        raise WorkerIOError("Unexpected message.")
+                    continue
+                return json.loads(data[:-1].decode())
+        except ConnectionResetError:
+            return None
+
+    def _make_connection(self) -> socket.socket:
+        '''Метод для создания подключения.'''
+        connection, parent_address = self._sock.accept()
+        return connection
+
+    def _check_connection(self, connection: socket.socket) -> bool:
+        '''Метод для проверки соединения путем отправки на сокет пустого
+        сообщения.'''
+        if connection is None:
+            return False
+
+        try:
+            connection.sendall(b'')
+            return True
+        except BrokenPipeError:
+            return False
+
+    def __del__(self) -> None:
+        if self._connection is not None:
+            self._connection.close()
+        self._sock.close()
+
+
+class Worker:
+    def __init__(self, wid: int, command: Command, datavars: Datavars):
+        self._wid: int = wid
+        self._datavars: Datavars = datavars
+
+        self._socket_path: str = f"./worker_{self._wid}.sock"
+        if os.path.exists(self._socket_path):
+            os.remove(self._socket_path)
+        self._input_queue: list = []
+
+    def run(self):
+        io = IOModule(self._socket_path)
+        title = io.receive()
+        print(title['msg'])
+        while True:
+            msg = "What is your desire?"
+            print(">", msg)
+            answer = io.input(msg)
+            print(f'> {answer}')
+            if answer == "stop":
+                break
+            elif answer == "output":
+                msg = "What kind of output you wanna see?"
+                print(">", msg)
+                answer = io.input(msg)
+                io.set_info(answer)
+            else:
+                msg = "I am sorry, but I could not help you("
+                print(">", msg)
+                io.output(msg)
+        print('STOPPED')
+
+    def initialize(self, io: IOModule):
+        pass
+
+    def run_command(self):
+        pass
+
+
+class DeprecatedWorker:
+    def __init__(self, wid, loop, sockets_path: str = 'calculate/server/'):
+        self._wid = wid
+        self._event_loop = loop
+
+        # Создаем сокет для взаимодействия воркера и сервера.
+        socket_path = os.path.join(sockets_path, f'worker_{self._wid}')
+        if os.path.exists(socket_path):
+            os.remove(socket_path)
+        self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+        self._socket.bind(socket_path)
+
     @staticmethod
     def _get_output(output_queue: Queue) -> dict:
         return output_queue.get()

calculate/templates/template_engine.py

@@ -1,8 +1,13 @@
 # vim: fileencoding=utf-8
 #
 from jinja2.ext import Extension
-from jinja2 import Environment, FileSystemLoader, TemplateSyntaxError, nodes,\
+from jinja2 import (
-                   contextfunction
+    Environment,
+    FileSystemLoader,
+    TemplateSyntaxError,
+    nodes,
+    contextfunction
+    )
 from jinja2.utils import missing
 from jinja2.runtime import Context, Undefined
 from collections.abc import MutableMapping
@@ -13,13 +18,27 @@ import copy
 import re
 import os
 
-from ..utils.package import PackageAtomParser, PackageAtomError, NOTEXIST,\
+from ..utils.package import (
-                            Version
+    PackageAtomParser,
-from ..utils.files import join_paths, check_directory_link, check_command,\
+    PackageAtomError,
-                          FilesError
+    NOTEXIST,
-from calculate.variables.datavars import HashType, NamespaceNode,\
+    Version
-                                         VariableNode, IniType, IntegerType,\
+    )
-                                         FloatType, ListType
+from ..utils.files import (
+    join_paths,
+    check_directory_link,
+    check_command,
+    FilesError
+    )
+from calculate.variables.datavars import (
+    HashType,
+    NamespaceNode,
+    VariableNode,
+    IniType,
+    IntegerType,
+    FloatType,
+    ListType
+    )
 from calculate.variables.loader import Datavars
 
 

calculate/templates/template_processor.py

@@ -1,16 +1,40 @@
 # vim: fileencoding=utf-8
 #
 from pprint import pprint
-from ..utils.package import PackageAtomParser, Package, PackageNotFound,\
+from ..utils.package import (
-                            PackageAtomName, Version, NonePackage
+    PackageAtomParser,
-from ..utils.files import join_paths, write_file, read_file_lines, FilesError,\
+    Package,
-                          check_directory_link, read_link, Process,\
+    PackageNotFound,
-                          get_target_from_link, get_directory_contents
+    PackageAtomName,
-from .template_engine import TemplateEngine, Variables, ConditionFailed,\
+    Version,
-                             ParametersProcessor, DIR, FILE,\
+    NonePackage,
-                             ParametersContainer
+    )
-from calculate.variables.datavars import StringType, ListType, NamespaceNode,\
+from ..utils.files import (
-                                         VariableNode, TableType
+    join_paths,
+    write_file,
+    read_file_lines,
+    FilesError,
+    check_directory_link,
+    read_link,
+    Process,
+    get_target_from_link,
+    get_directory_contents,
+    )
+from .template_engine import (
+    TemplateEngine,
+    Variables,
+    ConditionFailed,
+    ParametersProcessor,
+    DIR, FILE,
+    ParametersContainer,
+    )
+from calculate.variables.datavars import (
+    StringType,
+    ListType,
+    NamespaceNode,
+    VariableNode,
+    TableType,
+    )
 from calculate.variables.loader import Datavars
 from .format.base_format import Format
 from ..utils.io_module import IOModule

calculate/utils/fs.py

@@ -1,18 +1,21 @@
 import os
 from os.path import exists
 
+
 def readlink(path):
     try:
         return os.readlink(path)
     except FileNotFoundError:
         return None
 
+
 def readFileLines(path):
     if exists(path):
         with open(path, 'r') as f:
             for line in f:
                 yield line.rstrip("\n")
 
+
 def readFile(path):
     if exists(path):
         with open(path, 'r') as f:

calculate/utils/package.py

@@ -5,8 +5,15 @@ import re
 import glob
 from collections import OrderedDict
 from .files import read_file, read_link, join_paths, FilesError
-from pyparsing import Literal, Regex, Word, nums, alphanums,\
+from pyparsing import (
-                      LineEnd, SkipTo
+    Literal,
+    Regex,
+    Word,
+    nums,
+    alphanums,
+    LineEnd,
+    SkipTo,
+    )
 from jinja2 import PackageLoader, Environment
 from calculate.utils.tools import Singleton
 import hashlib

calculate/variables/loader.py

@@ -5,17 +5,39 @@ import logging
 import importlib
 import importlib.util
 from jinja2 import Environment, FileSystemLoader
-from calculate.variables.datavars import NamespaceNode, VariableNode,\
+from calculate.variables.datavars import (
-                                         ListType, IntegerType,\
+    NamespaceNode,
-                                         FloatType, IniType, TableType,\
+    VariableNode,
-                                         Namespace, HashType,\
+    ListType,
-                                         VariableNotFoundError, VariableError
+    IntegerType,
+    FloatType,
+    IniType,
+    TableType,
+    Namespace,
+    HashType,
+    VariableNotFoundError,
+    VariableError,
+    )
 from calculate.utils.gentoo import ProfileWalker
 from calculate.utils.files import read_file, FilesError
 from calculate.utils.tools import Singleton
-from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
+from pyparsing import (
-                      empty, printables, OneOrMore, lineno, line, SkipTo,\
+    Literal,
-                      LineEnd, Combine, nums
+    Word,
+    ZeroOrMore,
+    Group,
+    Optional,
+    restOfLine,
+    empty,
+    printables,
+    OneOrMore,
+    lineno,
+    line,
+    SkipTo,
+    LineEnd,
+    Combine,
+    nums,
+    )
 from enum import Enum
 from contextlib import contextmanager
 

run_templates.py

@@ -21,9 +21,12 @@ def main():
                         help="atom name of a build package.")
     parser.add_argument('-u', '--uninstall', type=str,
                         help="atom name of a uninstalling package.")
+
     args = parser.parse_args()
+
     datavars = Datavars()
     io_module = IOModule()
+
     if args.package == 'None':
         package = NonePackage
     else:

tests/server/test_server.py

@@ -15,8 +15,11 @@ test_client = TestClient(server.app)
 @pytest.mark.server
 class TestServer:
     def test_to_make_testfiles(self):
-        shutil.copytree(os.path.join(TESTFILES_PATH, 'gentoo.backup'),
+        shutil.copytree(os.path.join(TESTFILES_PATH, 'var.backup'),
-                        os.path.join(TESTFILES_PATH, 'gentoo'),
+                        os.path.join(TESTFILES_PATH, 'var'),
+                        symlinks=True)
+        shutil.copytree(os.path.join(TESTFILES_PATH, 'etc.backup'),
+                        os.path.join(TESTFILES_PATH, 'etc'),
                         symlinks=True)
 
     def test_get_root_message(self):
@@ -47,12 +50,13 @@ class TestServer:
         assert response.status_code == 200
         assert response.json() == {"id": 0, "name": "command_0"}
 
-    def test_get_worker_message_by_wid(self):
+    # def test_get_worker_message_by_wid(self):
-        response = test_client.get("/workers/0")
+    #     response = test_client.get("/workers/0")
-        assert response.status_code == 200
+    #     assert response.status_code == 200
-        data = response.json()
+    #     data = response.json()
-        assert data == {'type': 'log', 'level': 'INFO',
+    #     assert data == {'type': 'log', 'level': 'INFO',
-                        'msg': 'recieved message INFO'}
+    #                     'msg': 'recieved message INFO'}
 
     def test_for_removing_testfiles(self):
-        shutil.rmtree(os.path.join(TESTFILES_PATH, 'gentoo'))
+        shutil.rmtree(os.path.join(TESTFILES_PATH, 'var'))
+        shutil.rmtree(os.path.join(TESTFILES_PATH, 'etc'))

tests/server/testfiles/var.backup/db/pkg/test-category/test-package-1.0/SLOT

@@ -0,0 +1 @@
+1

tests/server/testfiles/var.backup/lib/calculate/config-archive/etc/dir_6/file_0

@@ -0,0 +1,3 @@
+options {
+    parameter-0 yes;
+};

tests/server/testfiles/var.backup/lib/calculate/config-archive/etc/file_1

@@ -0,0 +1,3 @@
+options {
+    parameter-0 yes;
+};

tests/server/testfiles/var.backup/lib/calculate/config-archive/etc/file_12

@@ -0,0 +1,3 @@
+section-name {
+    parameter-1 no;
+};

tests/server/testfiles/var.backup/lib/calculate/config-archive/etc/file_2

@@ -0,0 +1,3 @@
+options {
+    parameter-0 yes;
+};

tests/server/testfiles/variables/os/gentoo/__init__.py

@@ -20,8 +20,9 @@ Variable('make_profile', type=StringType, source='/etc/portage/make.profile')
 with Namespace('profile'):
     # Абсолютный путь до профиля
     Variable('path', type=StringType,
-             source=os.path.join(TESTFILES_PATH,
+             source=os.path.join(
-                                 "gentoo/repos/distros/profiles/CLD/amd64"))
+                            TESTFILES_PATH,
+                            "var/lib/gentoo/repos/distros/profiles/CLD/amd64"))
 
     def get_profile_name(path, repositories):
         profile_path = path.value
@@ -48,10 +49,10 @@ with Namespace('profile'):
 Variable('repositories', type=TableType,
          source=[{'name': 'distros',
                   'path': os.path.join(TESTFILES_PATH,
-                                       "gentoo/repos/distros")},
+                                       "var/lib/gentoo/repos/distros")},
                  {'name': 'calculate',
                   'path': os.path.join(TESTFILES_PATH,
-                                       "gentoo/repos/calculate")},
+                                       "var/lib/gentoo/repos/calculate")},
                  {'name': 'gentoo',
                   'path': os.path.join(TESTFILES_PATH,
-                                       "gentoo/portage")}])
+                                       "var/lib/gentoo/portage")}])