Browse Source

Added backgrounds format. fixed #75

master
Иванов Денис 1 year ago
parent
commit
e59a4511d6
  1. 318
      calculate/templates/format/backgrounds_format.py
  2. 20
      calculate/templates/format/contents_format.py
  3. 14
      calculate/templates/format/patch_format.py
  4. 357
      calculate/templates/template_engine.py
  5. 57
      calculate/templates/template_processor.py
  6. 18
      calculate/utils/files.py
  7. 143
      calculate/utils/images.py
  8. 2
      calculate/utils/package.py
  9. 42
      calculate/vars/install/os/__init__.py
  10. 2
      calculate/vars/install/os/func.py
  11. 9
      calculate/vars/main/__init__.py
  12. 5
      calculate/vars/main/cl/__init__.py
  13. 12
      calculate/vars/main/os/x11/__init__.py
  14. 15
      calculate/vars/update/cl/__init__.py
  15. 46
      conftest.py
  16. 3
      pytest.ini
  17. 331
      tests/templates/format/test_backgrounds.py
  18. 14
      tests/templates/format/test_contents.py
  19. 10
      tests/templates/format/test_patch.py
  20. BIN
      tests/templates/format/testfiles/backgrounds/picture_0.png
  21. BIN
      tests/templates/format/testfiles/backgrounds/picture_1.jpg
  22. BIN
      tests/templates/format/testfiles/backgrounds/picture_2.png
  23. BIN
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_10-1024x768.png
  24. BIN
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_10-1650x1050.png
  25. 1
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_10.md5sum
  26. BIN
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_11-1024x768.png
  27. BIN
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_11-1650x1050.png
  28. 1
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_11.md5sum
  29. 1
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_12.md5sum
  30. BIN
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_13-1024x768.png
  31. BIN
      tests/templates/format/testfiles/backgrounds/tmp.backup/result_13-1650x1050.png
  32. 3405
      tests/templates/test_directory_processor.py
  33. 6017
      tests/templates/test_template_executor.py
  34. 2410
      tests/templates/test_template_wrapper.py
  35. BIN
      tests/templates/testfiles/test_dir_processor_root/etc.backup/dir_80/image-300x100.jpeg
  36. BIN
      tests/templates/testfiles/test_dir_processor_root/etc.backup/dir_80/image-320x180.jpeg
  37. 1
      tests/templates/testfiles/test_dir_processor_root/etc.backup/dir_80/image.md5sum
  38. BIN
      tests/templates/testfiles/test_dir_processor_root/etc.backup/dir_82/image-300x100.jpeg
  39. BIN
      tests/templates/testfiles/test_dir_processor_root/etc.backup/dir_82/image-320x180.jpeg
  40. 1
      tests/templates/testfiles/test_dir_processor_root/etc.backup/dir_82/image.md5sum
  41. BIN
      tests/templates/testfiles/test_dir_processor_root/etc.backup/picture.jpg
  42. 2
      tests/templates/testfiles/test_dir_processor_root/templates_66/install/.calculate_directory
  43. 1
      tests/templates/testfiles/test_dir_processor_root/templates_66/install/dir_79/.calculate_directory
  44. 3
      tests/templates/testfiles/test_dir_processor_root/templates_66/install/dir_79/image
  45. 2
      tests/templates/testfiles/test_dir_processor_root/templates_67/install/.calculate_directory
  46. 1
      tests/templates/testfiles/test_dir_processor_root/templates_67/install/dir_80/.calculate_directory
  47. 4
      tests/templates/testfiles/test_dir_processor_root/templates_67/install/dir_80/image
  48. 2
      tests/templates/testfiles/test_dir_processor_root/templates_68/install/.calculate_directory
  49. 1
      tests/templates/testfiles/test_dir_processor_root/templates_68/install/dir_81/.calculate_directory
  50. 3
      tests/templates/testfiles/test_dir_processor_root/templates_68/install/dir_81/image
  51. 2
      tests/templates/testfiles/test_dir_processor_root/templates_69/install/.calculate_directory
  52. 1
      tests/templates/testfiles/test_dir_processor_root/templates_69/install/dir_82/.calculate_directory
  53. 4
      tests/templates/testfiles/test_dir_processor_root/templates_69/install/dir_82/image
  54. 122
      tests/utils/test_images.py
  55. 4
      tests/variables/test_datavars.py

318
calculate/templates/format/backgrounds_format.py

@ -0,0 +1,318 @@
# vim: fileencoding=utf-8
#
from ..template_engine import ParametersContainer, Variables
from ...variables.datavars import NamespaceNode, VariableNotFoundError
from ...variables.loader import Datavars
from ...utils.images import ImageMagick
from .base_format import Format, FormatError
from typing import Union, List, Tuple, NoReturn
import hashlib
import re
import os
class BackgroundsFormat(Format):
FORMAT = 'backgrounds'
EXECUTABLE = True
FORMAT_PARAMETERS = {'convert', 'stretch'}
def __init__(self, template_text: str,
template_path: str,
parameters: ParametersContainer,
datavars: Union[Datavars, NamespaceNode, Variables],
chroot_path: str = "/"):
self._lines: List[str] = [line for line
in template_text.strip().split('\n') if line]
self._datavars: Union[Datavars, NamespaceNode, Variables] = datavars
if parameters.source:
self._source = parameters.source
else:
raise FormatError("source parameter is not set for template with"
"with 'backgrounds' format.")
self._mirror: bool = parameters.mirror
self._stretch: bool = parameters.stretch
self._convert: Union[bool, str] = parameters.convert
self._empty_name: bool = (parameters.name == '')
# Измененные файлы.
self.changed_files: dict = dict()
# Список предупреждений.
self._warnings: list = []
# Флаг для тестов.
self._fake_chroot: bool = False
try:
self._fake_chroot = datavars.main.fake_chroot
except Exception:
pass
def execute_format(self, target_path: str, chroot_path: str = '/') -> dict:
'''Метод для запуска работы формата.'''
if not self._check_source(self._source, target_path, self._mirror):
return self.changed_files
print(f"FAKE_CHROOT: {self._fake_chroot}")
self._magician = ImageMagick(
chroot=chroot_path if not self._fake_chroot else "/")
source_resolution = self._magician.get_image_resolution(self._source)
resolutions_from_var = False
resolutions = self._get_resolutions_from_lines(self._lines,
source_resolution)
if not resolutions:
resolutions = self._get_resolutions_from_var(self._datavars)
resolutions_from_var = True
action_md5sum = self._get_action_md5sum(self._source, resolutions)
if not self._check_target_directory(target_path, self._mirror,
resolutions, action_md5sum):
return {}
images_format = self._get_format_value(self._convert,
self._datavars)
if self._convert and self._convert == "GFXBOOT":
converter = self._magician.convert_resize_gfxboot
else:
converter = self._magician.convert_resize_crop_center
output_paths = self._make_output_paths(self._lines, images_format,
target_path, resolutions,
resolutions_from_var)
# Выполняем преобразование изображений в требуемый размер и формат.
for resolution, output_path in output_paths.items():
if not self._check_stretch(source_resolution,
resolution,
self._stretch):
continue
width, height = resolution
converter(self._source, output_path, height, width,
image_format=images_format)
if output_path in self.changed_files:
self.changed_files[output_path] = 'M'
else:
self.changed_files[output_path] = 'N'
self._create_md5sum_file(target_path, action_md5sum)
return self.changed_files
def _get_format_value(self, convert: Union[bool, str],
datavars: Union[Datavars, NamespaceNode, Variables]
) -> str:
"""Метод для получения значения формата."""
if convert:
if convert == "JPG" or convert == "GFXBOOT":
images_format = "JPEG"
else:
images_format = convert
else:
images_format = self._magician.get_image_format(self._source)
return images_format
def _make_output_paths(self, lines: List[str],
images_format: str,
target_path: str,
resolutions: List[Tuple[int, int]],
resolutions_from_var: bool
) -> List[str]:
"""Метод для получения списка путей, по которым будут создаваться
изображения."""
if not self._empty_name:
path, name = os.path.split(target_path)
else:
path = os.path.dirname(target_path)
name = ''
paths = {}
if not resolutions_from_var and len(resolutions) == 1:
if self._empty_name:
raise FormatError("'name' parameter is empty in 'backgrounds'"
" format template with single file output.")
paths = {next(iter(resolutions)): target_path}
return paths
for width, height in resolutions:
paths[(width, height)] = (f"{path}/{name}{width}x{height}"
f".{images_format.lower()}")
return paths
def _get_resolutions_from_lines(self, lines: List[str],
source_resolution: Tuple[int, int]
) -> List[Tuple[int, int]]:
"""Метод для получения списка кортежей с разрешениями выходных
изображений из текста шаблона."""
resolutions = []
if lines:
for line in lines:
if (line.strip() == "original"
and source_resolution not in resolutions):
resolutions.append(source_resolution)
else:
try:
width, height = line.lower().strip().split("x")
resolution = (int(width), int(height))
except Exception:
raise FormatError("can not parse line from template"
" with 'backgrounds' format:"
f" '{line}'")
if resolution not in resolutions:
resolutions.append(resolution)
return resolutions
def _get_resolutions_from_var(self, datavars: Union[Datavars,
NamespaceNode,
Variables]
) -> List[Tuple[int, int]]:
"""Метод для получения списка кортежей с разрешениями выходных
изображений из переменной."""
try:
resolutions = []
for resolution in self._datavars.main.cl_resolutions:
resolution = tuple(resolution.strip().split("x"))
resolutions.append(resolution)
return resolutions
except VariableNotFoundError:
raise FormatError("resolutions values was not found.")
except Exception:
raise FormatError("can not use resolution values from variable:"
" 'main.cl_resolutions'")
def _check_target_directory(self, target_path: str, mirror: bool,
resolutions: List[Tuple[int, int]],
action_md5sum: str
) -> Union[bool, str]:
"""Метод для проверки содержимого целевой директории, удаления
изображений и md5-сумм, подлежащих удалению, а также сравнения
имеющейся в директории md5-суммы с суммой, полученной для действия
текущего шаблона."""
if not self._empty_name:
path, name = os.path.split(target_path)
else:
path = os.path.dirname(target_path)
name = ''
name_pattern = re.compile(rf"^{name}\d+x\d+")
md5sum = None
images = []
# Проверяем содержимое целевой директории.
for node in os.scandir(path):
if node.is_file():
if (node.name ==
f"{name.strip('-_.')}{'.' if name else ''}md5sum"):
md5sum = node.name
elif (node.name == name
or (name_pattern.search(node.name) is not None)):
images.append(node.name)
if not images and md5sum is None:
# Если нет файла суммы и нет изображений -- продолжаем выполнение
# шаблона.
return True
elif not images and md5sum is not None:
# Если есть файл суммы, но нет изображений -- удаляем файл суммы.
md5sum_path = os.path.join(path, md5sum)
os.unlink(md5sum_path)
self.changed_files[md5sum_path] = 'D'
return True
elif images and md5sum is None:
# Если есть файлы изображений, но нет суммы -- удаляем файлы
# изображений.
for image in images:
image_path = os.path.join(path, image)
os.unlink(image_path)
self.changed_files[image_path] = 'D'
return True
else:
# Сравниваем суммы из md5sum и для текущего действия если они
# сходятся, то делать ничего не надо, если нет -- удаляем
# имеющиеся изображения и md5-сумму.
with open(os.path.join(path, md5sum), "r") as md5sum_file:
current_md5sum = md5sum_file.read().strip()
if current_md5sum != action_md5sum:
for image in images:
image_path = os.path.join(path, image)
os.unlink(image_path)
self.changed_files[image_path] = 'D'
md5sum_path = os.path.join(path, md5sum)
os.unlink(md5sum_path)
self.changed_files[md5sum_path] = 'D'
return True
else:
return False
def _check_source(self, source: str, target_path: str, mirror: bool
) -> bool:
"""Метод для проверки исходного изображения."""
if not self._empty_name:
path, name = os.path.split(target_path)
else:
path = os.path.dirname(target_path)
name = ''
name_pattern = re.compile(rf"^{name}\d+x\d+")
if not os.path.exists(source):
if mirror:
for node in os.scandir(path):
if (node.name == name
or name_pattern.search(node.name) is not None
or node.name == (f"{node.name.strip('_-.')}"
f"{ '.' if name else '' }md5sum")):
os.unlink(node.path)
self.changed_files[node.path] = "D"
return False
else:
raise FormatError("image from 'source' parameter does not"
f" exist: {source}.")
return True
def _get_action_md5sum(self, source: str,
resolutions: List[Tuple[int, int]]) -> bool:
"""Метод для получения md5-суммы текущего действия шаблона,
рассчитываемой из последовательности байтов изображения и списка
разрешений, в которые данный файл должен быть конвертирован."""
print("RESOLUTIONS:", resolutions)
with open(source, "rb") as source_file:
md5_object = hashlib.md5(source_file.read())
for width, height in resolutions:
resolution = f"{width}x{height}"
md5_object.update(resolution.encode())
return md5_object.hexdigest()
def _create_md5sum_file(self, target_path: str, action_md5sum: str
) -> NoReturn:
"""Метод для создания файла с md5-суммой действия, выполненного
данным шаблоном."""
if not self._empty_name:
path, name = os.path.split(target_path)
else:
path = os.path.dirname(target_path)
name = ''
md5sum_path = (f"{path}/{name.strip('_-.')}"
f"{ '.' if name else '' }md5sum")
with open(md5sum_path, "w") as md5sum_file:
md5sum_file.write(action_md5sum)
if md5sum_path in self.changed_files:
self.changed_files[md5sum_path] = 'M'
else:
self.changed_files[md5sum_path] = 'N'
def _check_stretch(self, source_resolution: Tuple[int, int],
resolution: Tuple[int, int],
stretch: bool) -> bool:
"""Метод определяющий необходимость растягивания исходного изображения
и делающий вывод о том, возможно ли создание изображения заданного
разрешения исходя из значения параметра stretch."""
return (stretch
or (source_resolution[0] >= resolution[0]
and source_resolution[1] >= resolution[1]))
@property
def warnings(self):
return self._warnings

20
calculate/templates/format/contents_format.py

@ -2,11 +2,15 @@
#
import os
from .base_format import Format, FormatError
from calculate.utils.files import join_paths
from pyparsing import Literal, Regex, SkipTo, LineEnd, lineno, LineStart
from calculate.utils.package import PackageAtomParser, Package, PackageNotFound
from calculate.utils.files import join_paths
from glob import iglob
from ..template_engine import ParametersContainer, Variables
from ...variables.datavars import NamespaceNode
from ...variables.loader import Datavars
from fnmatch import fnmatch
from typing import Union
from glob import iglob
ADD, REMOVE, MOVE = range(0, 3)
@ -25,7 +29,8 @@ class ContentsFormat(Format):
def __init__(self, template_text: str,
template_path: str,
ignore_comments=None):
parameters: ParametersContainer,
datavars: Union[Datavars, NamespaceNode, Variables]):
self._command_methods = {ADD: self._add_command,
REMOVE: self._remove_command,
MOVE: self._move_command}
@ -39,7 +44,10 @@ class ContentsFormat(Format):
self._packages = dict()
self._atom_parser = None
def execute_format(self, target_path, chroot_path='/'):
# Предупреждения.
self._warnings: list = []
def execute_format(self, target_path: str, chroot_path='/') -> dict:
'''Метод для запуска работы формата.'''
self._package = dict()
self._atom_parser = PackageAtomParser(chroot_path=chroot_path)
@ -215,3 +223,7 @@ class ContentsFormat(Format):
if not result:
return [None]
return {'error': result, 'lineno': lineno(location, string)}
@property
def warnings(self):
return self._warnings

14
calculate/templates/format/patch_format.py

@ -3,6 +3,10 @@
from .base_format import Format
from calculate.utils.files import Process
from calculate.templates.format.base_format import FormatError
from ..template_engine import ParametersContainer, Variables
from ...variables.datavars import NamespaceNode
from ...variables.loader import Datavars
from typing import Union
from os import path
@ -12,7 +16,8 @@ class PatchFormat(Format):
def __init__(self, patch_text: str,
template_path: str,
ignore_comments=None):
parameters: ParametersContainer,
datavars: Union[Datavars, NamespaceNode, Variables]):
self._patch_text = patch_text
self._cwd_path = '/'
self._last_level = 0
@ -20,6 +25,9 @@ class PatchFormat(Format):
# Измененные файлы.
self.changed_files = dict()
# Предупреждения.
self._warnings: list = []
def execute_format(self, target_path, chroot_path='/'):
'''Метод для запуска работы формата.'''
self._cwd_path = target_path
@ -81,3 +89,7 @@ class PatchFormat(Format):
def __bool__(self):
return bool(self._patch_text)
@property
def warnings(self):
return self._warnings

357
calculate/templates/template_engine.py

@ -1,12 +1,15 @@
# vim: fileencoding=utf-8
#
from jinja2.ext import Extension
from jinja2.lexer import Token
from jinja2.parser import Parser
from jinja2 import (
Environment,
FileSystemLoader,
TemplateSyntaxError,
nodes,
contextfunction
contextfunction,
Template,
)
from jinja2.utils import missing
from jinja2.runtime import Context, Undefined
@ -22,7 +25,10 @@ from typing import (
Union,
Any,
List,
Tuple
Tuple,
NoReturn,
Optional,
Iterator,
)
from ..utils.package import (
PackageAtomName,
@ -39,6 +45,7 @@ from ..utils.files import (
FilesError
)
from calculate.variables.datavars import (
VariableNotFoundError,
HashType,
NamespaceNode,
VariableNode,
@ -55,7 +62,6 @@ import calculate.templates.template_filters as template_filters
# Типы шаблона: директория или файл.
DIR, FILE, LINK = range(3)
# Словарь, в котором можно регистрировать фильтры.
CALCULATE_FILTERS = {"cut": template_filters.cut}
@ -79,19 +85,23 @@ class ConditionFailed(TemplateSyntaxError):
class Variables(MutableMapping):
'''Класс-заглушка вместо модуля переменных для тестов.'''
def __init__(self, *args, **kwargs):
self.__attrs = dict(*args, **kwargs)
self.__attrs: dict = dict(*args, **kwargs)
self.__iter: Union[Iterator, None] = None
def __next__(self):
iterator = iter(self.__attrs)
return next(iterator)
def __next__(self) -> Any:
if self._iter is None:
self._iter = iter(self.__attrs)
return next(self._iter)
def __getattribute__(self, name: str):
def __getattribute__(self, name: str) -> Any:
if name == '_Variables__attrs':
return super().__getattribute__(name)
if name == 'available_packages':
return super().__getattribute__(name)
if name == '_variables':
return self.__attrs
if name == '_iter':
return self._iter
try:
return self.__attrs[name]
except KeyError:
@ -103,16 +113,16 @@ class Variables(MutableMapping):
packages.update({'custom'})
return packages
def __getitem__(self, name: str):
def __getitem__(self, name: str) -> Any:
return self.__attrs[name]
def __setitem__(self, name: str, value) -> None:
def __setitem__(self, name: str, value: Any) -> NoReturn:
self.__attrs[name] = value
def __delitem__(self, name: str) -> None:
def __delitem__(self, name: str) -> NoReturn:
del self.__attrs[name]
def __iter__(self):
def __iter__(self) -> Iterator:
return iter(self.__attrs)
def __len__(self) -> int:
@ -124,7 +134,7 @@ class Variables(MutableMapping):
def __contains__(self, name: str) -> bool:
return name in self.__attrs
def __hash__(self):
def __hash__(self) -> int:
return hash(id(self))
@ -135,7 +145,8 @@ class ParametersProcessor:
'format', 'unbound', 'mirror', 'run', 'exec',
'env', 'package', 'merge', 'postmerge',
'action', 'rebuild', 'restart', 'stop',
'start', 'handler', 'notify', 'group'}
'start', 'handler', 'notify', 'group',
'convert', 'stretch'}
inheritable_parameters: set = {'chmod', 'chown', 'autoupdate', 'env',
'package', 'action', 'handler', 'group'}
@ -158,21 +169,24 @@ class ParametersProcessor:
r'([r-][w-][Xx-])([r-][w-][Xx-])([r-][w-][Xx-])')
def __init__(self,
parameters_container: Union["ParametersContainer",
None] = None,
parameters_container: Optional["ParametersContainer"] = None,
chroot_path: str = '/',
datavars_module: Union[Datavars, Variables] = Variables(),
for_package: Union[Package, None] = None):
datavars_module: Union[Datavars,
NamespaceNode,
Variables] = Variables(),
for_package: Optional[Package] = None):
self.chroot_path: str = chroot_path
self.template_type: int = DIR
self.datavars_module: Union[Datavars, Variables] = datavars_module
self.datavars_module: Union[Datavars,
NamespaceNode,
Variables] = datavars_module
self._parameters_container: ParametersContainer = parameters_container
self.package_atom_parser: PackageAtomParser =\
PackageAtomParser(chroot_path=chroot_path)
self.package_atom_parser: PackageAtomParser = PackageAtomParser(
chroot_path=chroot_path)
self._groups: dict = {}
try:
@ -212,7 +226,9 @@ class ParametersProcessor:
'merge': self.check_merge_parameter,
'format': self.check_format_parameter,
'handler': self.check_handler_parameter,
'notify': self.check_notify_parameter
'notify': self.check_notify_parameter,
'convert': self.check_convert_parameter,
'stretch': self.check_stretch_parameter,
})
# Если добавляемый параметр должен быть проверен после того, как
@ -225,7 +241,9 @@ class ParametersProcessor:
'autoupdate': self.check_postparse_autoupdate,
'run': self.check_postparse_run,
'exec': self.check_postparse_exec,
'handler': self.check_postparse_handler
'handler': self.check_postparse_handler,
'convert': self.check_postparse_convert,
'stretch': self.check_postparse_stretch,
})
# Если параметр является наследуемым только при некоторых условиях --
@ -234,7 +252,7 @@ class ParametersProcessor:
def set_parameters_container(self,
parameters_container: "ParametersContainer"
) -> None:
) -> NoReturn:
'''Метод для установки текущего контейнера параметров.'''
self._parameters_container = parameters_container
self._added_parameters = set()
@ -244,7 +262,7 @@ class ParametersProcessor:
return self._for_package
@for_package.setter
def for_package(self, package: Package):
def for_package(self, package: Package) -> NoReturn:
self._for_package = package
def __getattr__(self, parameter_name: str) -> Any:
@ -258,7 +276,7 @@ class ParametersProcessor:
def check_template_parameter(self, parameter_name: str,
parameter_value: Any,
template_type: int, lineno: int) -> None:
template_type: int, lineno: int) -> NoReturn:
'''Метод, проверяющий указанный параметр.'''
self.lineno = lineno
self.template_type = template_type
@ -293,7 +311,7 @@ class ParametersProcessor:
self._parameters_container.set_parameter({parameter_name:
checked_value})
def check_postparse_parameters(self) -> None:
def check_postparse_parameters(self) -> NoReturn:
'''Метод, запускающий проверку параметров после их разбора.'''
for parameter, parameter_checker in\
self.postparse_checkers_list.items():
@ -306,7 +324,7 @@ class ParametersProcessor:
result)
def check_template_parameters(self, parameters: dict,
template_type: int, lineno: int) -> None:
template_type: int, lineno: int) -> NoReturn:
'''Метод, запускающий проверку указанных параметров.'''
self.template_type = template_type
self.lineno = lineno
@ -339,7 +357,6 @@ class ParametersProcessor:
parameter_name=checked_value)
# Методы для проверки параметров во время разбора шаблона.
def check_package_parameter(self, parameter_value: Any) -> str:
if not isinstance(parameter_value, str):
raise IncorrectParameter("'package' parameter must have value of"
@ -402,17 +419,17 @@ class ParametersProcessor:
raise IncorrectParameter(
"'restart' parameter value is not correct")
def check_stop_parameter(self, parameter_value: Any):
def check_stop_parameter(self, parameter_value: Any) -> str:
if not parameter_value and isinstance(parameter_value, bool):
raise IncorrectParameter("'stop' parameter value is empty")
return parameter_value
def check_start_parameter(self, parameter_value: Any):
def check_start_parameter(self, parameter_value: Any) -> str:
if not parameter_value and isinstance(parameter_value, bool):
raise IncorrectParameter("'start' parameter value is empty")
return parameter_value
def check_run_parameter(self, parameter_value: Any):
def check_run_parameter(self, parameter_value: Any) -> str:
if self.template_type == DIR:
raise IncorrectParameter("'run' parameter is not available in"
" directory templates")
@ -425,7 +442,7 @@ class ParametersProcessor:
" found")
return interpreter_path
def check_exec_parameter(self, parameter_value: Any):
def check_exec_parameter(self, parameter_value: Any) -> str:
if self.template_type == DIR:
raise IncorrectParameter("'exec' parameter is not available in"
" directory templates")
@ -438,7 +455,7 @@ class ParametersProcessor:
" found")
return interpreter_path
def check_chown_parameter(self, parameter_value: Any):
def check_chown_parameter(self, parameter_value: Any) -> dict:
if not parameter_value or isinstance(parameter_value, bool):
raise IncorrectParameter("'chown' parameter value is empty.")
parameter_value = self.get_chown_values(parameter_value)
@ -478,7 +495,8 @@ class ParametersProcessor:
x_mask = x_mask + "0"
return (int(chmod, 2), int(x_mask, 2))
def check_source_parameter(self, parameter_value: Any):
def check_source_parameter(self, parameter_value: Any
) -> Union[str, Tuple[bool, str]]:
if not parameter_value or isinstance(parameter_value, bool):
raise IncorrectParameter("'source' parameter value is empty")
@ -510,7 +528,7 @@ class ParametersProcessor:
return os.path.normpath(real_path)
def check_env_parameter(self, parameter_value: Any):
def check_env_parameter(self, parameter_value: Any) -> Union[None, set]:
env_set = set()
for env_value in parameter_value.split(','):
@ -538,20 +556,20 @@ class ParametersProcessor:
return env_set
def check_force_parameter(self, parameter_value: Any):
def check_force_parameter(self, parameter_value: Any) -> bool:
if isinstance(parameter_value, bool):
return parameter_value
else:
raise IncorrectParameter("'force' parameter value is not bool")
def check_autoupdate_parameter(self, parameter_value: Any):
def check_autoupdate_parameter(self, parameter_value: Any) -> bool:
if isinstance(parameter_value, bool):
return parameter_value
else:
raise IncorrectParameter(
"'autoupdate' parameter value is not bool.")
def check_format_parameter(self, parameter_value: Any):
def check_format_parameter(self, parameter_value: Any) -> str:
if self.template_type == DIR:
raise IncorrectParameter("'format' parameter is redundant for"
" directory templates.")
@ -564,13 +582,13 @@ class ParametersProcessor:
raise IncorrectParameter("'format' parameter must be string value not"
f" {type(parameter_value)}.")
def check_handler_parameter(self, parameter_value: Any):
def check_handler_parameter(self, parameter_value: Any) -> str:
if not isinstance(parameter_value, str):
raise IncorrectParameter("'handler' parameter must be string"
f" value not {type(parameter_value)}.")
return parameter_value
def check_notify_parameter(self, parameter_value: Any):
def check_notify_parameter(self, parameter_value: Any) -> List[str]:
if isinstance(parameter_value, list):
return parameter_value
elif isinstance(parameter_value, str):
@ -579,9 +597,33 @@ class ParametersProcessor:
raise IncorrectParameter("'notify' parameter must be string or list"
f" value not {type(parameter_value)}.")
# Методы для проверки параметров после разбора всего шаблона.
def check_convert_parameter(self, parameter_value: Any) -> str:
if not isinstance(parameter_value, str):
raise IncorrectParameter("'convert' parameter value must be string"
f" not '{type(parameter_value)}'.")
parameter_value = parameter_value.strip().upper()
try:
available_image_formats =\
self.datavars_module.main.cl_image_formats
except VariableNotFoundError:
# TODO возможно стоит кидать ошибку.
available_image_formats = ["JPEG", "PNG", "GIF", "JPG"]
if parameter_value not in available_image_formats:
raise IncorrectParameter(f"'{parameter_value}' image format is "
"not available. Available image formats: "
f"'{', '.join(available_image_formats)}.'"
)
return parameter_value
def check_stretch_parameter(self, parameter_value: Any) -> bool:
if not isinstance(parameter_value, bool):
raise IncorrectParameter("'stretch' parameter value should be bool"
f" value not '{type(parameter_value)}'")
return parameter_value
def check_postparse_append(self, parameter_value):
# Методы для проверки параметров после разбора всего шаблона.
def check_postparse_append(self, parameter_value: str) -> NoReturn:
if parameter_value == 'link':
if 'source' not in self._parameters_container:
raise IncorrectParameter("append = 'link' without source "
@ -595,7 +637,7 @@ class ParametersProcessor:
raise IncorrectParameter("'append' parameter is not 'compatible' "
"with the 'exec' parameter")
def check_postparse_run(self, parameter_value):
def check_postparse_run(self, parameter_value: str) -> NoReturn:
if self._parameters_container.append:
raise IncorrectParameter("'run' parameter is not 'compatible' "
"with the 'append' parameter")
@ -604,7 +646,7 @@ class ParametersProcessor:
raise IncorrectParameter("'run' parameter is not 'compatible' "
"with the 'exec' parameter")
def check_postparse_exec(self, parameter_value):
def check_postparse_exec(self, parameter_value: str) -> NoReturn:
if self._parameters_container.append:
raise IncorrectParameter("'exec' parameter is not 'compatible' "
"with the 'append' parameter")
@ -613,13 +655,16 @@ class ParametersProcessor:
raise IncorrectParameter("'exec' parameter is not 'compatible' "
"with the 'run' parameter")
def check_postparse_source(self, parameter_value):
def check_postparse_source(self,
parameter_value: Union[str, Tuple[bool, str]]
) -> NoReturn:
# Если файл по пути source не существует, но присутствует параметр
# mirror -- пропускаем шаблон для того, чтобы целевой файл мог быть
# удален в исполнительном модуле.
if isinstance(parameter_value, tuple):
if (self._parameters_container.append == "link" and
self._parameters_container.force):
if ((self._parameters_container.append == "link" and
self._parameters_container.force)
or self._parameters_container.format == "backgrounds"):
self._parameters_container['source'] = parameter_value[1]
elif not self._parameters_container.mirror:
raise IncorrectParameter(
@ -632,12 +677,12 @@ class ParametersProcessor:
"append = 'link' for directory template")
)
def check_postparse_autoupdate(self, parameter_value):
def check_postparse_autoupdate(self, parameter_value: bool) -> NoReturn:
if self._parameters_container.unbound:
raise IncorrectParameter("'unbound' parameter is incompatible"
" with 'autoupdate' parameter")
def check_postparse_handler(self, parameter_value):
def check_postparse_handler(self, parameter_value: bool) -> NoReturn:
if self._parameters_container.merge:
raise IncorrectParameter("'merge' parameter is not available"
" in handler templates")
@ -646,7 +691,7 @@ class ParametersProcessor:
raise IncorrectParameter("'package' parameter is not available"
" in handler templates")
def check_postparse_package(self, parameter_value):
def check_postparse_package(self, parameter_value: str) -> NoReturn:
groups = []
package_atom = PackageAtomParser.parse_atom_name(parameter_value)
@ -679,7 +724,8 @@ class ParametersProcessor:
" does not match the template condition",
self.lineno if hasattr(self, 'lineno') else 0)
def _check_package_group(self, package: dict, group_packages: list):
def _check_package_group(self, package: dict, group_packages: list
) -> bool:
'''Метод для проверки соответствия описания пакета, заданного словарем,
какому-либо описанию пакета, заданного в переменных groups.'''
for group_package in group_packages:
@ -699,16 +745,28 @@ class ParametersProcessor:
return True
return False
def check_postparse_convert(self, parameter_value: str) -> NoReturn:
template_format = self._parameters_container.format
if not template_format or template_format != "backgrounds":
raise IncorrectParameter("'convert' parameter available for"
" 'backgrounds' format only.")
def check_postparse_stretch(self, parameter_value: str) -> NoReturn:
template_format = self._parameters_container.format
if not template_format or template_format != "backgrounds":
raise IncorrectParameter("'stretch' parameter available for"
" 'backgrounds' format only.")
# Методы для проверки того, являются ли параметры наследуемыми.
def is_chmod_inheritable(self, parameter_value):
def is_chmod_inheritable(self, parameter_value: str) -> bool:
chmod_regex = re.compile(r'\d+')
if chmod_regex.search(parameter_value):
return False
return True
def get_chown_values(self, chown: str):
def get_chown_values(self, chown: str) -> dict:
"""Получить значения uid и gid из параметра chown."""
if chown and ':' in chown:
user_name, group_name = chown.split(':')
@ -746,7 +804,7 @@ class ParametersProcessor:
raise IncorrectParameter("'chown' value '{0}' is not correct".
format(chown))
def get_uid_from_passwd(self, user_name: str):
def get_uid_from_passwd(self, user_name: str) -> int:
"""Функция для получения uid из chroot passwd файла."""
passwd_file_path = os.path.join(self.chroot_path, 'etc/passwd')
passwd_dictionary = dict()
@ -771,7 +829,7 @@ class ParametersProcessor:
raise FilesError("passwd file was not found in {}".
format(passwd_file_path))
def get_gid_from_group(self, group_name: str):
def get_gid_from_group(self, group_name: str) -> int:
"""Функция для получения gid из chroot group файла."""
group_file_path = os.path.join(self.chroot_path, 'etc/group')
group_dictionary = dict()
@ -797,7 +855,7 @@ class ParametersProcessor:
format(group_file_path))
@classmethod
def _inspect_formats_package(cls):
def _inspect_formats_package(cls) -> NoReturn:
'''Метод для определения множества доступных форматов и
предоставляемых ими параметров.'''
if cls.format_is_inspected:
@ -845,9 +903,13 @@ class ParametersProcessor:
cls.formats_inspected = True
def resolve_or_missing(context, key, missing=missing, env={}):
def resolve_or_missing(context: "CalculateContext",
key: str, missing=missing,
env: Optional[set] = None) -> Any:
'''Переопределение функции из для поиска значений переменных из jinja2.
Ищет переменные в datavars.'''
if env is None:
env = {}
datavars = context.parent['__datavars__']
if key in context.vars:
@ -871,7 +933,7 @@ class CalculateContext(Context):
сохранять их.'''
_env_set = set()
def resolve(self, key):
def resolve(self, key: str) -> Any:
if self._legacy_resolve_mode:
rv = resolve_or_missing(self, key,
env=self._env_set)
@ -881,7 +943,7 @@ class CalculateContext(Context):
return self.environment.undefined(name=key)
return rv
def resolve_or_missing(self, key):
def resolve_or_missing(self, key: str) -> Any:
if self._legacy_resolve_mode:
rv = self.resolve(key)
if isinstance(rv, Undefined):
@ -894,56 +956,56 @@ class CalculateContext(Context):
class ParametersContainer(MutableMapping):
'''Класс для хранения параметров, взятых из шаблона, и передачи
их шаблонизатору.'''
def __init__(self, parameters_dictionary=None):
def __init__(self, parameters_dictionary: Optional[dict] = None):
# Слой ненаследуемых параметров.
self.__parameters = {}
self.__parameters: dict = {}
# Слой наследуемых параметров.
if parameters_dictionary is not None:
self.__inheritable = parameters_dictionary
self.__inheritable: dict = parameters_dictionary
else:
self.__inheritable = {}
self.__inheritable: dict = {}
def set_parameter(self, item_to_add: dict):
def set_parameter(self, item_to_add: dict) -> NoReturn:
self.__parameters.update(item_to_add)
def set_inheritable(self, item_to_add: dict):
def set_inheritable(self, item_to_add: dict) -> NoReturn:
self.__inheritable.update(item_to_add)
def get_inheritables(self):
def get_inheritables(self) -> "ParametersContainer":
return ParametersContainer(copy.deepcopy(self.__inheritable))
def remove_not_inheritable(self):
def remove_not_inheritable(self) -> NoReturn:
self.__parameters.clear()
def print_parameters_for_debug(self):
def print_parameters_for_debug(self) -> NoReturn:
print('Parameters:')
pprint(self.__parameters)
print('Inherited:')
pprint(self.__inheritable)
def is_inherited(self, parameter_name):
def is_inherited(self, parameter_name: str) -> bool:
return (parameter_name not in self.__parameters
and parameter_name in self.__inheritable)
def remove_parameter(self, parameter_name):
def remove_parameter(self, parameter_name: str) -> NoReturn:
if parameter_name in self.__parameters:
self.__parameters.pop(parameter_name)
elif parameter_name in self.__inheritable:
self.__inheritable.pop(parameter_name)
def change_parameter(self, parameter, value):
def change_parameter(self, parameter: str, value: Any) -> NoReturn:
if parameter in self.__parameters:
self.__parameters.update({parameter: value})
elif parameter in self.__inheritable:
self.__inheritable.update({parameter: value})
def _clear_container(self):
def _clear_container(self) -> NoReturn:
self.__parameters.clear()
self.__inheritable.clear()
def __getattr__(self, parameter_name):
def __getattr__(self, parameter_name: str) -> Any:
if (parameter_name not in
ParametersProcessor.available_parameters):
raise IncorrectParameter("Unknown parameter: '{}'".
@ -956,7 +1018,7 @@ class ParametersContainer(MutableMapping):
else:
return False
def __getitem__(self, name):
def __getitem__(self, name: str) -> Any:
if name in self.__parameters:
return self.__parameters[name]
elif name in self.__inheritable:
@ -964,31 +1026,31 @@ class ParametersContainer(MutableMapping):
else:
return False
def __setitem__(self, name, value):
def __setitem__(self, name: str, value: Any) -> NoReturn:
self.__parameters[name] = value
def __delitem__(self, name):
def __delitem__(self, name: str) -> NoReturn:
if name in self.__parameters:
del self.__parameters[name]
if name in self.__inheritable:
del self.__inheritable[name]
def __iter__(self):
def __iter__(self) -> Iterator[str]:
return iter(set(self.__parameters).union(self.__inheritable))
def __len__(self):
def __len__(self) -> int:
return len(set(self.__parameters).union(self.__inheritable))
def __repr__(self):
def __repr__(self) -> str:
return '<ParametersContainer: parameters={0}, inheritables={1}>'.\
format(self.__parameters, self.__inheritable)
def __contains__(self, name):
def __contains__(self, name: str) -> bool:
return name in self.__parameters or name in self.__inheritable
@property
def parameters(self):
def parameters(self) -> dict:
return self.__parameters
@ -999,10 +1061,14 @@ class CalculateExtension(Extension):
# Виды операций в теге save.
ASSIGN, APPEND, REMOVE = range(3)
def __init__(self, environment, parameters_processor: ParametersProcessor,
datavars_module=Variables(), chroot_path="/"):
def __init__(self, environment: Environment,
parameters_processor: ParametersProcessor,
datavars_module: Union[Datavars,
NamespaceNode,
Variables] = Variables(),
chroot_path: str = "/"):
super().__init__(environment)
self.environment = environment
self.environment: Environment = environment
self.package_atom_parser = PackageAtomParser(chroot_path=chroot_path)
self.environment.globals.update({'pkg': self.pkg})
@ -1011,11 +1077,11 @@ class CalculateExtension(Extension):
self._datavars = datavars_module
self.parameters_processor = parameters_processor
self.template_type = DIR
self.template_type: int = DIR
# Флаг, указывающий, что тег calculate уже был разобран. Нужен для
# того, чтобы проверять единственность тега calculate.
self.calculate_parsed = False
self.calculate_parsed: bool = False
self.tags = {'calculate', 'save', 'set_var'}
self.CONDITION_TOKENS_TYPES = {'eq', 'ne', 'lt', 'gt', 'lteq', 'gteq'}
@ -1031,24 +1097,25 @@ class CalculateExtension(Extension):
self.parse_methods = {'calculate': self.parse_calculate,
'save': self.parse_save}
def __call__(self, env):
def __call__(self, env: Environment) -> "CalculateExtension":
# Необходимо для обеспечения возможности передать готовый объект
# расширения, а не его класс.
return self
def parse(self, parser):
def parse(self, parser: Parser) -> List[nodes.Output]:
self.parser = parser
self.stream = parser.stream
tag_token = self.stream.current.value
return [self.parse_methods[tag_token]()]
def parse_save(self):
def parse_save(self) -> nodes.Output:
'''Метод для разбора тега save, сохраняющего значение указанной
переменной datavars.'''
lineno = next(self.stream).lineno
target_file = nodes.Const('', lineno=lineno)
# Получаем имя целевого файла.
if self.stream.skip_if('dot'):
target_file_name = self.stream.expect('name').value
if target_file_name in self.TARGET_FILES_SET:
@ -1084,7 +1151,7 @@ class CalculateExtension(Extension):
raise TemplateSyntaxError("'=' is expected in 'save' tag",
lineno=lineno)
def parse_calculate(self):
def parse_calculate(self) -> nodes.Output:
'''Метод для разбора тега calculate, содержащего значения параметров и
условия выполнения шаблона.'''
lineno = next(self.stream).lineno
@ -1155,7 +1222,7 @@ class CalculateExtension(Extension):
self.calculate_parsed = True
return nodes.Output([nodes.Const('')], lineno=lineno)
def _is_variable_name(self, token):
def _is_variable_name(self, token: Token) -> bool:
'''Метод для проверки токена на предмет того, что он является частью
имени переменной.'''
if not token.type == 'name':
@ -1168,7 +1235,8 @@ class CalculateExtension(Extension):
return True
return False
def check_parameter(self, parameter_name, parameter_value, context):
def check_parameter(self, parameter_name: str, parameter_value: Any,
context: CalculateContext) -> str:
self.parameters_processor.check_template_parameter(
parameter_name,
parameter_value,
@ -1176,7 +1244,7 @@ class CalculateExtension(Extension):
self.stream.current.lineno)
return ''
def parse_condition(self):
def parse_condition(self) -> Template:
try:
condition_node = self.parser.parse_expression(with_condexpr=True)
condition_node = self.call_method(
@ -1196,7 +1264,7 @@ class CalculateExtension(Extension):
.format(str(error)),
lineno=self.stream.current.lineno)
def check_conditions(self, conditions: list):
def check_conditions(self, conditions: List[Template]) -> NoReturn:
for condition in conditions:
self.condition_result = False
try:
@ -1210,7 +1278,7 @@ class CalculateExtension(Extension):
lineno=self.stream.current.lineno)
# DEPRECATED
def get_condition_result(self):
def get_condition_result(self) -> bool:
'''Метод для разбора условий из тега calculate.'''
self.condition_result = False
@ -1233,13 +1301,14 @@ class CalculateExtension(Extension):
return self.condition_result
def set_condition_result(self, condition_result):
def set_condition_result(self, condition_result: Any) -> str:
'''Метод для сохранения результата вычисления условия.'''
self.condition_result = condition_result
return ''
def _make_save_node(self, variable_name_node, target_file_node, optype,
lineno):
def _make_save_node(self, variable_name_node: nodes.List,
target_file_node: nodes.Const, optype: int,
lineno: int) -> nodes.Output:
'''Метод для создания ноды, сохраняющей переменные.'''
right_value = self.parser.parse_expression(with_condexpr=True)
optype_node = nodes.Const(optype, lineno=lineno)
@ -1252,8 +1321,9 @@ class CalculateExtension(Extension):
lineno=lineno)
return nodes.Output([save_variable_node], lineno=lineno)
def save_variable(self, variable, right_value, target_file,
optype, context):
def save_variable(self, variable: List[str], right_value: Any,
target_file: str, optype: int,
context: CalculateContext) -> str:
'''Метод для сохранения значений переменных указанных в теге save.'''
datavars = context.parent['__datavars__']
if variable[0] not in datavars:
@ -1272,7 +1342,7 @@ class CalculateExtension(Extension):
# Теперь меняем знaчение переменной.
if isinstance(value_container, NamespaceNode):
self._modify_variables(variable, value_container, right_value,
optype, target=target_file,
optype, target_file=target_file,
modify_only=modify_only)
elif isinstance(value_container, VariableNode):
hash_value = value_container.get_value().get_hash()
@ -1296,7 +1366,7 @@ class CalculateExtension(Extension):
def _find_value_container(self, variable: List[str],
vars_package: NamespaceNode,
modify_only: bool = True
) -> Union[NamespaceNode]:
) -> Union[NamespaceNode, VariableNode]:
'''Метод для поиска контейнера, путь к которому указан в аргументе.
Этим контейнером может быть пространство имен или хэш.'''
current_container = vars_package
@ -1329,8 +1399,10 @@ class CalculateExtension(Extension):
current_container.get_fullname()))
return current_container
def _modify_variables(self, variable, namespace, new_value, optype,
target='', modify_only=True):
def _modify_variables(self, variable: List[str], namespace: NamespaceNode,
new_value: Any, optype: int,
target_file: Optional[str] = None,
modify_only: bool = True) -> NoReturn:
'''Метод для модификации значения переменной.'''
variable_name = variable[-1]
@ -1353,18 +1425,20 @@ class CalculateExtension(Extension):
raise SaveError("can not create variable '{}' in not 'custom'"
" namespace".format('.'.join(variable)))
if target:
if target_file:
if namespace._variables[variable_name].variable_type is HashType:
for key, value in new_value.items():
self._save_to_target(variable, key, value, target)
self._save_to_target(variable, key, value, target_file)
else:
self._save_to_target(variable[:-1], variable_name,
new_value, target)
new_value, target_file)
def _modify_hash(self, variable, hash_variable, new_value, optype,
target=''):
# DEPRECATED
def _modify_hash(self, variable_name: List[str],
hash_variable: VariableNode, new_value, optype,
target_file: Optional[str] = None) -> NoReturn:
'''Метод для модификации значения в переменной-хэше.'''
value_name = variable[-1]
value_name = variable_name[-1]
hash_value = hash_variable.get_value().get_hash()
if value_name in hash_value:
@ -1376,22 +1450,25 @@ class CalculateExtension(Extension):
hash_value.update({value_name: new_value})
hash_variable.set(hash_value)
if target:
self._save_to_target(variable[:-1], value_name,
new_value, target)
if target_file:
self._save_to_target(variable_name[:-1], value_name,
new_value, target_file)
def _save_to_target(self, namespace_name, variable_name, value, target):
def _save_to_target(self, namespace_name: List[str],
variable_name: str, value: Any, target_file: str
) -> NoReturn:
'''Метод для добавления переменной в список переменных, значение
которых было установлено через тег save и при этом должно быть
сохранено в указанном файле: save.target_file.'''
namespace_name = tuple(namespace_name)
target_file_dict = self._datavars.variables_to_save[target]
target_file_dict = self._datavars.variables_to_save[target_file]
if namespace_name not in target_file_dict:
target_file_dict.update({namespace_name: dict()})
target_file_dict[namespace_name].update(
{variable_name: ('=', str(value))})
def _append_variable_value(self, variable, value):
def _append_variable_value(self, variable: VariableNode,
value: Any) -> Any:
'''Метод описывающий операцию += в теге save.'''
variable_value = variable.get_value()
if (variable.variable_type is IntegerType or
@ -1429,7 +1506,8 @@ class CalculateExtension(Extension):
# значение.
return variable_value
def _remove_variable_value(self, variable, value):
def _remove_variable_value(self, variable: VariableNode, value: Any
) -> Any:
'''Метод описывающий операцию -= в теге save.'''
variable_value = variable.get_value()
@ -1469,7 +1547,7 @@ class CalculateExtension(Extension):
# значение.
return variable_value
def _get_parameter(self):