5
0
Fork 0

Server is partly implemented.

master
Иванов Денис 3 anos atrás
commit fa3c023c0d

@ -8,11 +8,11 @@ from typing import Tuple, Union, Any
class ParameterError(Exception):
pass
...
class ValidationError(ParameterError):
pass
...
class CyclicValidationError(ValidationError):
@ -372,7 +372,7 @@ class BaseParameter:
def validate(self, container, datavars, value) -> None:
'''Метод для проверки корректности параметра. Переопределяется при
создании нового типа параметра.'''
pass
return None
def validate_value(self, value):
'''Метод для запуска валидации параметра с использованием

@ -4,9 +4,15 @@ import re
import inspect
from typing import Callable, Any, Union, List, Generator
from calculate.templates.template_processor import DirectoryProcessor
from calculate.variables.datavars import DependenceAPI, DependenceError,\
VariableNode, NamespaceNode,\
HashType, TableType, StringType
from calculate.variables.datavars import (
DependenceAPI,
DependenceError,
VariableNode,
NamespaceNode,
HashType,
TableType,
StringType,
)
from calculate.variables.loader import Datavars
from calculate.utils.io_module import IOModule
from collections.abc import Iterable, Mapping

@ -76,6 +76,7 @@ class Server:
continue
return output
# Обработчики запросов серверу.
async def _get_root(self) -> dict:
'''Обработчик корневых запросов.'''
return {'msg': 'root msg'}
@ -99,7 +100,7 @@ class Server:
'name': f'command_{cid}'}
async def _get_worker(self, wid: int):
'''Тестовый '''
'''Тестовый обработчик.'''
self._make_worker(wid=wid)
worker = self._workers[wid]
worker.run(None)
@ -115,6 +116,9 @@ class Server:
pass
return
# Обработчики сообщений воркеров.
# Вспомогательные методы.
def _add_routes(self, method: Callable, routes: dict) -> None:
'''Метод для добавления методов.'''
for path, handler in routes.items():
@ -123,16 +127,15 @@ class Server:
def _make_worker(self, wid: int = None):
'''Метод для создания воркера для команды.'''
worker = Worker(self._event_loop)
if wid is None:
self._workers[wid] = worker
if wid is not None:
self._workers[wid] = Worker(wid, self._event_loop)
return wid
elif not self._workers:
self._workers[0] = worker
self._workers[0] = Worker(0, self._event_loop)
return 0
else:
wid = max(self._workers.keys()) + 1
self._workers[wid] = worker
self._workers[wid] = Worker(wid, self._event_loop)
return wid
def _make_command(self, command_id: str) -> int:

@ -1,21 +1,173 @@
import os
import json
import socket
import logging
from typing import Union
from ..variables.loader import Datavars
from multiprocessing import Queue, Process
from ..commands.commands import CommandRunner, Command
# from time import sleep
class Worker:
def __init__(self, loop):
self._output_queue = Queue()
self._in_queue = Queue()
self._event_loop = loop
class WorkerIOError(KeyboardInterrupt):
pass
class IOModule:
'''Класс модуля ввода/вывода для воркеров.'''
def __init__(self, socket_path: str):
self._sock = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
self._sock.bind(socket_path)
self._sock.listen()
self._connection = None
def input(self, msg: str) -> str:
'''Метод через который возможен ввод данных в скрипт.'''
input_request = json.dumps({"type": "input",
"msg": msg}) + '\0'
answer = None
while answer is None:
if not self._check_connection(self._connection):
self._connection = self._make_connection()
self._connection.sendall(input_request.encode())
answer = self._get()
return answer['data']
def set_debug(self, msg: str) -> None:
self.output(msg, level=logging.DEBUG)
def set_info(self, msg: str) -> None:
self.output(msg, level=logging.INFO)
async def send(self, data: dict):
self._in_queue.put(data)
def set_warning(self, msg: str) -> None:
self.output(msg, level=logging.WARNING)
async def get(self):
data = await self._event_loop.run_in_executor(None, self._get_output,
self._output_queue)
def set_error(self, msg: str) -> None:
self.output(msg, level=logging.ERROR)
def set_critical(self, msg: str) -> None:
self.output(msg, level=logging.CRITICAL)
def output(self, msg: str, level: int = logging.INFO) -> None:
'''Метод для отправки серверу вывода с указанием уровня.'''
if not self._check_connection(self._connection):
self._connection = self._make_connection()
output_request = json.dumps({"type": "output",
"level": level,
"msg": msg}) + '\0'
self._connection.sendall(output_request.encode())
def send(self, data: dict) -> None:
'''Метод для отправки данных серверу.'''
if not self._check_connection(self._connection):
self._connection = self._make_connection()
data = json.dumps(data) + '\0'
self._connection.sendall(data.encode())
def receive(self) -> dict:
'''Метод для получения данных от сервера.'''
data = None
while data is None:
if not self._check_connection(self._connection):
self._connection = self._make_connection()
data = self._get()
return data
def _get(self) -> Union[None, dict]:
'''Метод для считывания данных, возможно, поделенных на кадры.'''
try:
data = b''
while True:
chunk = self._connection.recv(1024)
if not chunk:
return None
data += chunk
if not data.endswith(b'\0'):
if b'\0' in data:
# Если после символа конца сообщения есть еще какие-то
# данные -- считаем их наличие ошибкой.
raise WorkerIOError("Unexpected message.")
continue
return json.loads(data[:-1].decode())
except ConnectionResetError:
return None
def _make_connection(self) -> socket.socket:
'''Метод для создания подключения.'''
connection, parent_address = self._sock.accept()
return connection
def _check_connection(self, connection: socket.socket) -> bool:
'''Метод для проверки соединения путем отправки на сокет пустого
сообщения.'''
if connection is None:
return False
try:
connection.sendall(b'')
return True
except BrokenPipeError:
return False
def __del__(self) -> None:
if self._connection is not None:
self._connection.close()
self._sock.close()
class Worker:
def __init__(self, wid: int, command: Command, datavars: Datavars):
self._wid: int = wid
self._datavars: Datavars = datavars
self._socket_path: str = f"./worker_{self._wid}.sock"
if os.path.exists(self._socket_path):
os.remove(self._socket_path)
self._input_queue: list = []
def run(self):
io = IOModule(self._socket_path)
title = io.receive()
print(title['msg'])
while True:
msg = "What is your desire?"
print(">", msg)
answer = io.input(msg)
print(f'> {answer}')
if answer == "stop":
break
elif answer == "output":
msg = "What kind of output you wanna see?"
print(">", msg)
answer = io.input(msg)
io.set_info(answer)
else:
msg = "I am sorry, but I could not help you("
print(">", msg)
io.output(msg)
print('STOPPED')
def initialize(self, io: IOModule):
pass
def run_command(self):
pass
class DeprecatedWorker:
def __init__(self, wid, loop, sockets_path: str = 'calculate/server/'):
self._wid = wid
self._event_loop = loop
# Создаем сокет для взаимодействия воркера и сервера.
socket_path = os.path.join(sockets_path, f'worker_{self._wid}')
if os.path.exists(socket_path):
os.remove(socket_path)
self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
self._socket.bind(socket_path)
@staticmethod
def _get_output(output_queue: Queue) -> dict:
return output_queue.get()

@ -1,8 +1,13 @@
# vim: fileencoding=utf-8
#
from jinja2.ext import Extension
from jinja2 import Environment, FileSystemLoader, TemplateSyntaxError, nodes,\
contextfunction
from jinja2 import (
Environment,
FileSystemLoader,
TemplateSyntaxError,
nodes,
contextfunction
)
from jinja2.utils import missing
from jinja2.runtime import Context, Undefined
from collections.abc import MutableMapping
@ -13,13 +18,27 @@ import copy
import re
import os
from ..utils.package import PackageAtomParser, PackageAtomError, NOTEXIST,\
Version
from ..utils.files import join_paths, check_directory_link, check_command,\
FilesError
from calculate.variables.datavars import HashType, NamespaceNode,\
VariableNode, IniType, IntegerType,\
FloatType, ListType
from ..utils.package import (
PackageAtomParser,
PackageAtomError,
NOTEXIST,
Version
)
from ..utils.files import (
join_paths,
check_directory_link,
check_command,
FilesError
)
from calculate.variables.datavars import (
HashType,
NamespaceNode,
VariableNode,
IniType,
IntegerType,
FloatType,
ListType
)
from calculate.variables.loader import Datavars

@ -1,16 +1,40 @@
# vim: fileencoding=utf-8
#
from pprint import pprint
from ..utils.package import PackageAtomParser, Package, PackageNotFound,\
PackageAtomName, Version, NonePackage
from ..utils.files import join_paths, write_file, read_file_lines, FilesError,\
check_directory_link, read_link, Process,\
get_target_from_link, get_directory_contents
from .template_engine import TemplateEngine, Variables, ConditionFailed,\
ParametersProcessor, DIR, FILE,\
ParametersContainer
from calculate.variables.datavars import StringType, ListType, NamespaceNode,\
VariableNode, TableType
from ..utils.package import (
PackageAtomParser,
Package,
PackageNotFound,
PackageAtomName,
Version,
NonePackage,
)
from ..utils.files import (
join_paths,
write_file,
read_file_lines,
FilesError,
check_directory_link,
read_link,
Process,
get_target_from_link,
get_directory_contents,
)
from .template_engine import (
TemplateEngine,
Variables,
ConditionFailed,
ParametersProcessor,
DIR, FILE,
ParametersContainer,
)
from calculate.variables.datavars import (
StringType,
ListType,
NamespaceNode,
VariableNode,
TableType,
)
from calculate.variables.loader import Datavars
from .format.base_format import Format
from ..utils.io_module import IOModule

@ -1,18 +1,21 @@
import os
from os.path import exists
def readlink(path):
try:
return os.readlink(path)
except FileNotFoundError:
return None
def readFileLines(path):
if exists(path):
with open(path, 'r') as f:
for line in f:
yield line.rstrip("\n")
def readFile(path):
if exists(path):
with open(path, 'r') as f:

@ -5,8 +5,15 @@ import re
import glob
from collections import OrderedDict
from .files import read_file, read_link, join_paths, FilesError
from pyparsing import Literal, Regex, Word, nums, alphanums,\
LineEnd, SkipTo
from pyparsing import (
Literal,
Regex,
Word,
nums,
alphanums,
LineEnd,
SkipTo,
)
from jinja2 import PackageLoader, Environment
from calculate.utils.tools import Singleton
import hashlib

@ -5,17 +5,39 @@ import logging
import importlib
import importlib.util
from jinja2 import Environment, FileSystemLoader
from calculate.variables.datavars import NamespaceNode, VariableNode,\
ListType, IntegerType,\
FloatType, IniType, TableType,\
Namespace, HashType,\
VariableNotFoundError, VariableError
from calculate.variables.datavars import (
NamespaceNode,
VariableNode,
ListType,
IntegerType,
FloatType,
IniType,
TableType,
Namespace,
HashType,
VariableNotFoundError,
VariableError,
)
from calculate.utils.gentoo import ProfileWalker
from calculate.utils.files import read_file, FilesError
from calculate.utils.tools import Singleton
from pyparsing import Literal, Word, ZeroOrMore, Group, Optional, restOfLine,\
empty, printables, OneOrMore, lineno, line, SkipTo,\
LineEnd, Combine, nums
from pyparsing import (
Literal,
Word,
ZeroOrMore,
Group,
Optional,
restOfLine,
empty,
printables,
OneOrMore,
lineno,
line,
SkipTo,
LineEnd,
Combine,
nums,
)
from enum import Enum
from contextlib import contextmanager

@ -21,9 +21,12 @@ def main():
help="atom name of a build package.")
parser.add_argument('-u', '--uninstall', type=str,
help="atom name of a uninstalling package.")
args = parser.parse_args()
datavars = Datavars()
io_module = IOModule()
if args.package == 'None':
package = NonePackage
else:

@ -15,8 +15,11 @@ test_client = TestClient(server.app)
@pytest.mark.server
class TestServer:
def test_to_make_testfiles(self):
shutil.copytree(os.path.join(TESTFILES_PATH, 'gentoo.backup'),
os.path.join(TESTFILES_PATH, 'gentoo'),
shutil.copytree(os.path.join(TESTFILES_PATH, 'var.backup'),
os.path.join(TESTFILES_PATH, 'var'),
symlinks=True)
shutil.copytree(os.path.join(TESTFILES_PATH, 'etc.backup'),
os.path.join(TESTFILES_PATH, 'etc'),
symlinks=True)
def test_get_root_message(self):
@ -47,12 +50,13 @@ class TestServer:
assert response.status_code == 200
assert response.json() == {"id": 0, "name": "command_0"}
def test_get_worker_message_by_wid(self):
response = test_client.get("/workers/0")
assert response.status_code == 200
data = response.json()
assert data == {'type': 'log', 'level': 'INFO',
'msg': 'recieved message INFO'}
# def test_get_worker_message_by_wid(self):
# response = test_client.get("/workers/0")
# assert response.status_code == 200
# data = response.json()
# assert data == {'type': 'log', 'level': 'INFO',
# 'msg': 'recieved message INFO'}
def test_for_removing_testfiles(self):
shutil.rmtree(os.path.join(TESTFILES_PATH, 'gentoo'))
shutil.rmtree(os.path.join(TESTFILES_PATH, 'var'))
shutil.rmtree(os.path.join(TESTFILES_PATH, 'etc'))

@ -20,8 +20,9 @@ Variable('make_profile', type=StringType, source='/etc/portage/make.profile')
with Namespace('profile'):
# Абсолютный путь до профиля
Variable('path', type=StringType,
source=os.path.join(TESTFILES_PATH,
"gentoo/repos/distros/profiles/CLD/amd64"))
source=os.path.join(
TESTFILES_PATH,
"var/lib/gentoo/repos/distros/profiles/CLD/amd64"))
def get_profile_name(path, repositories):
profile_path = path.value
@ -48,10 +49,10 @@ with Namespace('profile'):
Variable('repositories', type=TableType,
source=[{'name': 'distros',
'path': os.path.join(TESTFILES_PATH,
"gentoo/repos/distros")},
"var/lib/gentoo/repos/distros")},
{'name': 'calculate',
'path': os.path.join(TESTFILES_PATH,
"gentoo/repos/calculate")},
"var/lib/gentoo/repos/calculate")},
{'name': 'gentoo',
'path': os.path.join(TESTFILES_PATH,
"gentoo/portage")}])
"var/lib/gentoo/portage")}])

Carregando…
Cancelar
Salvar