Browse Source

Some utils and template processing modules is added.

packages
Иванов Денис 1 year ago
parent
commit
fcf41bcf8a
33 changed files with 2170 additions and 0 deletions
  1. +0
    -0
      calculate/templates/format/templates/__init__.py
  2. +746
    -0
      calculate/templates/template_processor.py
  3. +631
    -0
      calculate/utils/device.py
  4. +161
    -0
      calculate/utils/io_module.py
  5. +330
    -0
      calculate/utils/mount.py
  6. +150
    -0
      calculate/utils/tools.py
  7. +0
    -0
      calculate/vars/os/__init__.py
  8. +0
    -0
      tests/__init__.py
  9. +0
    -0
      tests/templates/format/testfiles/__init__.py
  10. +52
    -0
      tests/templates/test_directory_processor.py
  11. +8
    -0
      tests/templates/test_template_action.py
  12. +1
    -0
      tests/templates/testfiles/test_dir_1/test_dir/.calculate_directory
  13. +11
    -0
      tests/templates/testfiles/test_dir_1/test_dir/file.conf
  14. +3
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir1/.calculate_directory
  15. +1
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir1/template1/.calculate_directory
  16. +5
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir1/template1/template_1
  17. +1
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir1/template2/.calculate_directory
  18. +10
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir1/template2/a_template_2.xml
  19. +10
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir1/template2/b_template_2.xml
  20. +4
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir2/.calculate_directory
  21. +1
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir2/template3/.calculate_directory
  22. +11
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir2/template3/template_3.conf
  23. +2
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir3/.calculate_directory
  24. +1
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir3/template_4/.calculate_directory
  25. +4
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir3/template_4/template_file.conf
  26. +1
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir3/template_5/.calculate_directory
  27. +6
    -0
      tests/templates/testfiles/test_dir_1/test_dir/subdir3/template_5/template_file.conf
  28. +1
    -0
      tests/templates/testfiles/test_dir_2/test_dir/.calculate_directory
  29. +1
    -0
      tests/templates/testfiles/test_dir_2/test_dir/template_1/.calculate_directory
  30. +8
    -0
      tests/templates/testfiles/test_dir_2/test_dir/template_1/template_1.conf
  31. +1
    -0
      tests/templates/testfiles/test_dir_2/test_dir/template_2/.calculate_directory
  32. +9
    -0
      tests/templates/testfiles/test_dir_2/test_dir/template_2/template_2.conf
  33. +0
    -0
      tests/utils/__init__.py

+ 0
- 0
calculate/templates/format/templates/__init__.py View File


+ 746
- 0
calculate/templates/template_processor.py View File

@@ -0,0 +1,746 @@
import os
import sys
import re
import stat
import shutil
from .template_engine import TemplateEngine, Variables, ConditionFailed
from ..utils.io_module import IOModule
from collections import OrderedDict
from ..utils.mount import Mounts


class TemplateAction:
def __init__(self, output_module=IOModule(), base_directory='/'):
self.output = output_module
self.mounts = None
self.base_directory = base_directory
self.exec_list = OrderedDict()

self.DIRECTORY_APPENDS = {
'remove': self.remove_directory,
'clear': self.clear_directory,
'replace': self.remove_directory,
'link': self.link_directory,
'join': self.join_directory
}

self.FILE_APPENDS = {
'replace': self.replace_file,
'remove': self.remove_file,
'clear': self.clear_file,
'link': self.link_file,
'join': self.join_file,
'before': self.before_file,
'after': self.after_file
}

def process_template_directory(self, target_path, template_path,
parameters={}):
self.template_path = template_path

# разбираем общие параметры шаблона.
self.force = parameters.pop('force', False)
self.autoupdate = parameters.pop('autoupdate', False)

self.source = parameters.pop('source', False)
if self.source and parameters['append'] != 'link':
self.output.set_error(("'source' parameter is set without "
"'append = link' in template {}").
format(self.template_path))
return False

self.chown = parameters.get('chown', False)
if not self.get_chown_values(self.chown):
return False

self.chmod = parameters.get('chmod', False)

def join_directory(self, target_path, template_path):
if os.access(target_path, os.F_OK):
return True
directory = target_path
directories_to_create = []
while not os.access(directory, os.F_OK) and directory:
directory = os.path.split(directory)[0]
directories_to_create.append(directory)
try:
current_mode, current_uid, current_gid = self.get_file_info(
target_path,
'all')
except OSError:
self.output.set_error("No access to the directory: {}".
format(target_path))
return False
if self.chmode:
mode = self.chmode
else:
mode = current_mode

if self.chown['uid']:
uid = self.chown['uid']
else:
uid = current_uid

if self.chown['gid']:
gid = self.chown['gid']
else:
gid = current_gid

directories_to_create.reverse()
for directory in directories_to_create:
try:
if mode:
os.mkdir(directory)
os.chmod(directory, mode)
else:
os.mkdir(directory)
self.chown_directory(directory)
except OSError:
self.output.set_error('failed to create directory: {}'.
format(directory))
return False
return True

def remove_directory(self, target_path):
if os.path.isdir(target_path) or os.path.exists(target_path):
try:
shutil.rmtree(target_path)
return True
except Exception:
self.output.set_error("failed to delete the directory: {}".
format(target_path))
return False
else:
self.output.set_error("failed to delete the directory: {}".
format(target_path))
self.output.set_error(
"target file is not directory or not exists.".
format(target_path))
return False

def clear_directory(self, target_path):
if os.path.isdir(target_path) and os.path.exists(target_path):
for node in os.scandir(target_path):
if node.is_dir():
if self.remove_directory(node.path):
return True
else:
self.output.set_error('Failed to delete directory: {}'.
format(target_path))
else:
try:
os.remove(node.path)
return True
except OSError:
self.output.set_error('Failed to delete directory: {}'.
format(target_path))
else:
self.output.set_error('failed to delete directory: {}'.
format(target_path))
self.output.set_error(
'target file is not directory or does not exist: {}'.
format(target_path))


def link_directory(self, target_path):
if not self.source:
self.output.set_error(("'source' parameter is not defined for "
"'append = link' in template {}").
format(self.template_path))
return False

def replace_directory(self, target_path):
pass

def process_template_file(self, target_path, template_path,
template_text='', parameters={}):
self.template_text = template_text
self.template_path = template_path

# разбираем общие параметры шаблона.
self.autoupdate = parameters.pop('autoupdate', False)
self.force = parameters.pop('force', False)
self.mirror = parameters.pop('mirror', False)
self.protected = parameters.pop('protected', False)

self.format = parameters.pop('format', False)
self.source = parameters.pop('source', False)
self.run = parameters.pop('run', False)
self.exec = parameters.pop('exec', False)
self.chmod = parameters.pop('chmod', False)
self.chown = parameters.pop('chown', False)

if (self.source and parameters['append'] not in
{'link', 'join', 'after', 'before'}):
self.output.set_error(("'source' parameter is set without"
" a suitable 'append' parameter"
" in template {}").
format(self.template_path))
return False

try:
append_value = parameters.pop('append', False)
if not append_value:
self.output("'append' parameter is not defined in template {}".
format(self.template_path))
return False
self.append = self.DIRECTORY_APPENDS[append_value]
except KeyError:
self.output.set_error("Unknown 'append' value in template {}".
format(self.template_path))
return False

# получаем uid и gid значения из параметра chown.
if self.chown:
self.chown = self.get_chown_values()
if not self.chown:
return False

# временный параметр format будет заменен на автоопределение формата.
if not self.format:
self.output.set_error(
"'format' parameter is not defined in template {}".
format(self.template_path)
)
return False

def join_file(self, target_path):
pass

def before_file(self, target_path):
pass

def after_file(self, target_path):
pass

def replace_file(self, target_path):
pass

def remove_file(self, target_path):
try:
if os.path.islink(target_path):
try:
os.unlink(target_path)
return True
except OSError:
self.output.set_error('Template error: {}'.
format(target_path))
self.output.set_error('Failed to delete the link: {}'.
format(target_path))
return False
if os.path.isfile(target_path):
try:
os.remove(target_path)
return True
except OSError:
self.output.set_error('Template error: {}'.
format(target_path))
self.output.set_error('Failed to delete the file: {}'.
format(target_path))
finally:
pass

def clear_file(self, target_path):
try:
with open(target_path, 'w') as f:
f.truncate(0)
except IOError:
self.output.set_error("Template error: {}".
format(self.template_path))
self.output.set_error("Failed to clear the file: {}".
format(target_path))
return False

def link_file(self, target_path):
pass

def get_parameter_names(self):
'''Временная замена механизма получения множества
поддерживаемых форматами параметров.'''
parameters_set = {'name', 'path', 'append', 'chmod', 'chown',
'autoupdate', 'env', 'link', 'force', 'symbolic',
'format', 'comment', 'protected', 'mirror', 'run',
'exec', 'env', 'stretch', 'convert', 'dotall',
'multiline', 'dconf', 'package', 'merge',
'postmerge', 'action', 'rebuild'}

return parameters_set

# возможно, создать отдельный класс для хранения имен имеющихся в системе.
def get_chown_values(self, chown: str):
"""Получить значения uid и gid из параметра chown."""
if chown and ':' in chown:
user_name, group_name = chown.split(':')
import pwd

try:
if self.base_directory == '/':
uid = pwd.getpwnam(user_name).pw_uid
else:
uid = self.get_uid_from_passwd(user_name)
except (KeyError, TypeError):
self.output.set_error(
"There is no such user in the system: {}".
format(user_name))
self.output.set_error(
"Wrong 'chown' value '{0}' in the template: {1}".
format(chown, self.template_path))
return False

import grp

try:
if self.base_directory == '/':
gid = grp.getgrnam(group_name).pw_gid
else:
gid = self.get_gid_from_group(group_name)
except (KeyError, TypeError):
self.output.set_error(
"There is no such group in the system: {}".
format(group_name))
self.output.set_error(
"Wrong 'chown' value '{0}' in the template: {1}".
format(chown, self.template_path))
return False
return {'uid': uid, 'gid': gid}
else:
self.output.set_error(
"Wrong 'chown' value '{0}' in the template: {1}".
format(chown, self.template_path))
return False

def get_uid_from_passwd(self, user_name: str):
"""Взять uid из chroot passwd файла."""
passwd_file_path = os.path.join(self.base_directory, 'etc/passwd')
passwd_dictionary = []
if os.path.exists(passwd_file_path):
with open(passwd_file_path, 'r') as passwd_file:
for line in passwd_file:
if line.startswith('#'):
continue
passwd_item = tuple(line.split(':')[0:3:2])
if (len(passwd_item) > 1 and passwd_item[0]
and passwd_item[0]):
passwd_dictionary.append(passwd_item)
passwd_dictionary = dict(passwd_dictionary)
return int(passwd_dictionary[user_name])
else:
self.output.set_error("passwd file was not found in {}".
format(passwd_file_path))
return False

def get_gid_from_group(self, group_name: str):
"""Взять gid из chroot group файла."""
group_file_path = os.path.join(self.base_directory, 'etc/group')
group_dictionary = []
if os.path.exists(group_file_path):
with open(group_file_path, 'r') as group_file:
for line in group_file:
if line.startswith('#'):
continue
group_item = tuple(line.split(':')[0:3:2])
if len(group_item) > 1 and group_item[0] and group_item[1]:
group_dictionary.append(group_item)
group_dictionary = dict(group_dictionary)
if group_name in group_dictionary:
return int(group_dictionary[group_name])
else:
self.output.set_error("'{0}' gid was not found in {1}".
format(group_name, group_file_path))
else:
self.output.set_error("group file was not found in {}".
format(group_file_path))
return False

def chown_directory(self, target_path):
"""Сменить владельца директории."""
try:
os.chown(target_path, self.chown['uid'], self.chown['gid'])
except (OSError, Exception) as error:
# возможно потребуются дополнительные проверки.
self.output.set_error('Can not chown directory: {}'.
format(target_path))
return False

def chmod_directory(self, target_path):
"""Сменить права доступа к директории."""
try:
os.chmod(target_path, self.chmod)
except (OSError, Exception) as error:
# возможно потребуются дополнительные проверки.
self.output.set_error('Can not chmod directory: {}'.
format(target_path))
return False

def chown_file(self, target_path, check_existation=True):
"""Сменить владельца файла."""
try:
if check_existation and not os.path.exists(target_path):
open(target_path, 'w').close()
os.lchown(target_path, self.chown['uid'], self.chown['gid'])
return True
except (OSError, Exception) as error:
# возможно потребуются дополнительные проверки.
self.output.set_error('Can not chown file: {}'.format(target_path))
return False

def chmod_file(self, target_path, check_existation=True):
"""Сменить права доступа к директории."""
try:
if check_existation and not os.path.exists(target_path):
open(target_path, 'w').close()
os.chmod(target_path, self.chmod)
return True
except (OSError, Exception) as error:
# возможно потребуются дополнительные проверки.
self.output.set_error('Can not chmod file: {}'.format(target_path))
return False

def get_file_info(self, path, info='all'):
file_stat = os.stat(path)
if info == 'all':
return stat.S_IMODE(file_stat.st_mode), file_stat.st_uid,\
file_stat.st_gid
if info == 'mode':
return stat.S_IMODE(file_stat.st_mode)
if info == 'owner':
return file_stat.st_uid, file_stat.st_gid

def is_vfat(self, path):
if self.mounts is None:
self.mounts = Mounts()
if self.mounts.get_from_fstab(what=self.mounts.TYPE,
where=self.mounts.DIR,
is_in=path) in ('vfat', 'ntfs-3g',
'ntfs'):
return True
return False

def check_os_error(self, error, path):
if hasattr(error, 'errno') and error.errno == os.errno.EPERM:
if self.is_vfat(path):
return True
if hasattr(error, 'errno') and error.errno == os.errno.EACCES and\
'var/calculate/remote' in path:
return True
return False


class FormatDetector:
pass


class DirectoryProcessor:
chmod_regex = re.compile(r'\d{3}')

def __init__(self, action, datavars_module=Variables(), package='',
output_module=IOModule()):
self.INHERITABLE_PARAMETERS = {'chmod', 'chown', 'autoupdate',
'env', 'package', 'action'}
self.datavars_module = datavars_module
self.output = output_module
self.template_action = TemplateAction(output_module=self.output)

self.action = action
self.chroot_path = datavars_module.main.cl_chroot.path
self.parameters_set = self.template_action.get_parameter_names()
self.template_engine = TemplateEngine(
datavars_module=datavars_module,
parameters_set=self.parameters_set
)
self.template_paths = (self.datavars_module.
main.cl_template.path.split(','))

self.for_package = package
self.processed_packages = []

self.packages_to_merge = []
self.packages_file_paths = {}

def process_template_directories(self):
# Проходим каталоги из main.cl_template.path
for directory_path in self.template_paths:
directory_path = directory_path.strip()
entries = os.scandir(directory_path)
for node in entries:
self.walk_directory_tree(node.path,
self.chroot_path,
{}, set())

if self.for_package:
self.output.set_info('Processing packages from merge parameter...')
self.processed_packages.append(self.for_package)
self.merge_packages()

def merge_packages(self):
not_merged_packages = []
while self.packages_to_merge:
self.for_package = self.packages_to_merge.pop()
if self.for_package not in self.packages_file_paths:
self.output.set_error(
"Error: package '{0}' not found for action '{1}'.".
format(self.for_package, self.action)
)
not_merged_packages.append(self.for_package)
continue
for template_path in self.packages_file_paths[self.for_package]:
base_directory, queue = self.get_directories_queue(
template_path
)
first_directory = queue.pop()
first_directory_path = os.path.join(base_directory,
first_directory)
self.walk_directory_tree(first_directory_path,
self.chroot_path, {}, set(),
directories_queue=queue)

self.processed_packages.append(self.for_package)
if not_merged_packages:
self.output.set_error('Packages {} is not merged.'.
format(','.join(self.packages_to_merge)))
else:
self.output.set_success('All packages are merged...')

def get_directories_queue(self, path):
directories_queue = []
for base_dir in self.template_paths:
base_dir = base_dir.strip()
if path.startswith(base_dir):
base_directory = base_dir
break

while path != base_directory:
path, dir_name = os.path.split(path)
directories_queue.append(dir_name)

return base_directory, directories_queue

def walk_directory_tree(self, current_directory_path, target_path,
directory_parameters, directory_env,
directories_queue=[]):
template_files = []
template_directories = []
directory_name = os.path.basename(current_directory_path)

current_env = directory_env.copy()
current_target_path = target_path

directory_parameters = directory_parameters.copy()

entries = os.scandir(current_directory_path)
self.template_engine.change_directory(current_directory_path)
for node in entries:
# Временное, вместо этого добавить переменную ignore_files.
if node.name.endswith('.swp'):
continue
if node.is_dir():
template_directories.append(node.path)
elif node.is_file():
template_files.append(node.name)

# обрабатываем в первую очередь шаблон директории.
if '.calculate_directory' in template_files:
template_files.remove('.calculate_directory')

try:
self.template_engine.process_template('.calculate_directory',
env=current_env)
except ConditionFailed:
self.output.set_warning('Condition failed in {}'.
format(current_directory_path))
self.output.console_output.print_default('\n')
return
except Exception as error:
self.output.set_error('Template error: {}'.
format(str(error)))
return

directory_parameters.update(self.template_engine.parameters)

# Если есть параметр name -- меняем имя текущего каталога.
if 'name' in directory_parameters:
directory_name = directory_parameters['name']

# Если есть параметр path -- меняем текущий путь к целевому
# каталогу.
if 'path' in directory_parameters:
current_target_path = directory_parameters['path']

if directory_parameters.get('append', None) != 'skip':
current_target_path = os.path.join(current_target_path,
directory_name)
if directory_parameters.get('action', None) != self.action:
self.output.set_error(
'Action parameter not found or not relevant: {}'.
format(directory_name))
self.output.console_output.print_default('\n')
return

# Если шаблоны обрабатываются для одного пакета -- проверяем
# параметр package и собираем пути к директориям других
# пакетов, необходимые для merge.
if self.for_package:
if 'package' not in directory_parameters:
self.output.set_warning(
"'package' is not defined for {}".
format(current_directory_path)
)
return
elif directory_parameters['package'] != self.for_package:
package_name = directory_parameters['package']
if package_name in self.packages_file_paths:
self.packages_file_paths[package_name].append(
current_directory_path
)
else:
self.packages_file_paths[package_name] =\
[current_directory_path]
self.output.set_warning(
"'package' parameter is not actual in {}".
format(current_directory_path)
)
return

# Если параметр env содержит несуществующий модуль -- шаблон не
# выполняем.
if 'env' in directory_parameters:
env_to_check = directory_parameters['env'].split(',')
for name in env_to_check:
if name.strip() not in self.datavars_module:
return
del(directory_parameters['env'])

# Если есть параметр merge -- собираем указанные в нем пакеты
# в списке имен пакетов для последующей обработки.
if self.for_package and 'merge' in directory_parameters:
merge_list = directory_parameters['merge'].split(',')
del(directory_parameters['merge'])
for package in merge_list:
self.packages_to_merge.append(package.strip())

# Выполняем действие с директорией.
self.output.set_success('Processing directory: {}'.
format(current_directory_path))
self.output.console_output.print_default(
'Directory parameters: {}\n'.
format(directory_parameters)
)
self.output.set_info(
'Actions using template directory. Target: {}'.
format(current_target_path)
)

# Оставляем только наследуемые параметры.
directory_parameters = {name: value for name, value in
directory_parameters.items()
if name in self.INHERITABLE_PARAMETERS}
if 'chmod' in directory_parameters:
match = self.chmod_regex.search(
str(directory_parameters['chmod'])
)
if match:
del(directory_parameters['chmod'])

if directories_queue:
next_node = directories_queue.pop()
if next_node in template_files:
template_files = [next_node]
template_directories = []
else:
next_directory_path = os.path.join(current_directory_path,
next_node)
template_files = []
template_directories = [next_directory_path]

# обрабатываем файлы шаблонов хранящихся в директории.
for template_name in template_files:
template_parameters = {}
template_parameters = directory_parameters.copy()
try:
self.template_engine.process_template(template_name,
env=current_env.copy())
except ConditionFailed:
self.output.set_warning('Condition failed for: {}'.
format(template_name))
continue
except Exception as error:
self.output.set_error('Template error: {}'.
format(str(error)))
continue

template_parameters.update(self.template_engine.parameters)
template_text = self.template_engine.template_text

if template_parameters.get('action', None) != self.action:
self.output.set_warning(
'Action parameter not found or not relevant.'
)
continue

if 'action' in template_parameters:
del(template_parameters['action'])

if self.for_package:
if 'package' not in template_parameters:
self.output.set_warning("'package' is not defined for {}".
format(template_name))
continue
elif template_parameters['package'] != self.for_package:
package_name = template_parameters['package']
template_path = os.path.join(current_directory_path,
template_name)
if package_name in self.packages_file_paths:
self.packages_file_paths[package_name].append(
template_path
)
else:
self.packages_file_paths[package_name] =\
[template_path]
self.output.set_warning(
"'package' parameter is not actual in {}".
format(template_name)
)
continue

if 'merge' in template_parameters:
merge_list = template_parameters['merge'].split(',')
del(template_parameters['merge'])
for package_name in merge_list:
if package_name not in self.processed_packages:
self.packages_to_merge.append(package_name.strip())

# Если параметр env содержит несуществующий модуль -- шаблон не
# выполняем.
if 'env' in template_parameters:
env_to_check = template_parameters['env'].split(',')
for name in env_to_check:
if name.strip() not in self.datavars_module:
continue
del(template_parameters['env'])

if 'name' in template_parameters:
template_name = template_parameters['name']

if 'path' in template_parameters:
target_file_path = os.path.join(template_parameters['path'],
template_name)
else:
target_file_path = os.path.join(current_target_path,
template_name)

# Выполняем действие с использованием шаблона.
self.output.set_success('Processing template: {}...'.
format(template_name))
self.output.console_output.print_default(
'With parameters: {}\n'.
format(template_parameters)
)
self.output.set_info('Target file: {}.\nTemplate text:'.
format(target_file_path))
self.output.console_output.print_default(template_text + '\n\n')

# проходимся далее по директориям.
for directory_path in template_directories:
self.walk_directory_tree(directory_path, current_target_path,
directory_parameters, current_env,
directories_queue=directories_queue)

+ 631
- 0
calculate/utils/device.py View File

@@ -0,0 +1,631 @@
import os
import re
from . import files
from .tools import Cachable, GenericFS, unique
from time import sleep
from pprint import pprint


def get_uuid_dict(reverse=False, devices=()):
'''Получить словарь со значениями UUID блочных устройств.'''
blkid_process = files.Process('/sbin/blkid', '-s', 'UUID',
'-c', '/dev/null', *devices)
re_split = re.compile('^([^:]+):.*UUID="([^"]+)"', re.S)

output_items = {}
search_results = []
lines = blkid_process.read_lines()
for line in lines:
search_result = re_split.search(line)
if search_result:
search_results.append(search_result.groups())

output_items = (("UUID={}".format(uuid), udev.get_devname(name=dev,
fallback=dev))
for dev, uuid in search_results)
if reverse:
return {v: k for k, v in output_items}
else:
return {k: v for k, v in output_items}


def find_device_by_partition(device):
'''Найти устройство, к которому относится данный раздел.'''
info = udev.get_device_info(name=device)
if info.get('DEVTYPE', '') != 'partition':
return ''
parent_path = os.path.dirname(info.get('DEVPATH', ''))
if parent_path:
device_info = udev.get_device_info(path=parent_path)
return device_info.get('DEVNAME', '')
return False


def count_partitions(device_name):
'''Посчитать количество разделов данного устройства.'''
syspath = udev.get_device_info(name=device_name).get('DEVPATH', '')
if not syspath:
return 0
device_name = os.path.basename(syspath)
return len([x for x in sysfs.listdir(syspath) if x.startswith(device_name)])


def get_lspci_output(filter_name=None, short_info=False):
'''Функция для чтения вывода lspci. Возвращает словарь с идентификаторами
устройств и информацией о них.'''
re_data = re.compile(r'(\S+)\s"([^"]+)"\s+"([^"]+)"\s+"([^"]+)"', re.S)
lspci_column_names = ('type', 'vendor', 'name')
if filter_name:
if callable(filter_name):
filter_function = filter_name
else:
def filter_function(input):
return filter_name in input
else:
def filter_function(input):
return input
lspci_path = files.check_utils('/usr/sbin/lspci')
lspci_output = files.Process(lspci_path, '-m').read_lines()
output = {}
lspci_output = list(filter(filter_function, lspci_output))
for line in lspci_output:
search_result = re_data.search(line)
if search_result:
search_result = search_result.groups()
device_id = search_result[0]
output[device_id] = {key: value for key, value in zip(
lspci_column_names,
search_result[1:])}
return output


class LvmCommand:
'''Класс для работы с командой lvm и выполнения с ее помощью различных
действий.'''
@property
def lvm_command(self):
'''Возвращает кешированный путь к lvm.'''
return files.get_program_path('/sbin/lvm')

def get_physical_extent_size(self):
if not self.lvm_command:
return ''
pv_data = files.Process(self.lvm_command, 'lvmconfig', '--type',
'full', 'allocation/physical_extent_size')
if pv_data.success():
return pv_data.read()
return ''

def get_pvdisplay_output(self, option):
'''Получить вывод pv_display c указанной option.'''
if not self.lvm_command:
return ''
pv_data = files.Process(self.lvm_command, 'pvdisplay', '-C', '-o',
option, '--noh')
if pv_data.success():
return pv_data.read()
return False

def vg_scan(self):
'''Найти существующие в системе группы томов.'''
if self.lvm_command:
return files.Process(self.lvm_command, 'vgscan').success()
return False

def vg_change(self):
'''Изменить атрибуты группы томов.'''
if self.lvm_command:
failed = files.Process('vgchange', '-ay').failed()
failed |= files.Process('vgchange', '--refresh').failed()
return not failed
return False

def lv_change(self, groups):
'''Изменить атрибуты логического тома.'''
if self.lvm_command:
failed = False
for group in groups:
failed |= files.Process('lvchange', '-ay', group).failed()
failed |= files.Process('lvchange', '--refresh',
group).failed()
return failed
return False

def execute(self, *command):
'''Выполнить указанную LVM команду.'''
if self.lvm_command:
return files.Process(self.lvm_command, *command).success()
return False

def double_execute(self, *command):
'''Выполнить дважды указанную LVM команду.'''
if not self.execute(*command):
sleep(2)
return self.execute(*command)
return False

def remove_lv(self, vg, lv):
'''Удалить указанный логических томов из системы.'''
return self.double_execute('lvremove', '{0}/{1}'.format(vg, lv), '-f')

def remove_vg(self, vg):
'''Удалить указанную группу томов из системы.'''
return self.double_execute('vgremove', vg, '-f')

def remove_pv(self, pv):
'''Удалить LVM метку с физического тома.'''
return self.double_execute('pvremove', pv, '-ffy')


class Lvm(Cachable):
'''Класс для получения информации о lvm.'''
def __init__(self, lvm_command):
super().__init__()
self.lvm_command = lvm_command

def get_pvdisplay_full(self):
'''Получить полный вывод команды pvdisplay.'''
get_pvdisplay_output = self.get_pvdisplay_output(
'vg_name,lv_name,pv_name'
).strip()
if get_pvdisplay_output:
output = (line.split() for line in
get_pvdisplay_output.split('\n'))
return [tuple(y.strip() for y in x)
for x in output if x and len(x) == 3]

@property
def get_volume_groups(self):
'''Получить имеющиеся в системе группы томов.'''
return sorted({vg for vg, lv, pv in self.get_pvdisplay_full()})

@Cachable.method_cached()
def get_physical_extent_size(self):
'''Получить размер физического диапазона.'''
return self.lvm_command.get_physical_extent_size()

@Cachable.method_cached()
def get_pvdisplay_output(self, option):
'''Получить с помощью pvdisplay информацию .'''
return self.lvm_command.get_pvdisplay_output(option)

def get_used_partitions(self, vg_condition, lv_condition):
'''Получить испоьзуемые разделы.'''
return list(sorted({part for vg, lv, part in self.get_pvdisplay_full()
if vg == vg_condition and lv == lv_condition}))

def refresh(self):
if not os.environ.get('EBUILD_PHASE', False):
self.lvm_command.vg_scan()
self.lvm_command.vg_change()
self.lvm_command.lv_change(lvm)

def remove_lv(self, vg, lv):
return self.lvm_command.remove_lv(vg, lv)

def remove_vg(self, vg):
return self.lvm_command.remove_vg(vg)

def remove_pv(self, pv):
return self.lvm_command.remove_pv(pv)


class MdadmCommand:
'''Класс для работы с командой mdadm.'''
@property
def mdadm_command(self):
'''Возвращает кешированный путь к mdamd.'''
return files.get_program_path('/sbin/mdadm')

def stop_raid(self, raid_device):
'''Остановить устройство из RAID-массива.'''
if not self.mdadm_command:
return False
return files.Process(self.mdadm_command, '-S', raid_device).success()

def zero_superblock(self, device):
'''Затереть superblock для составляющей RAID-массива.'''
if not self.mdadm_command:
return False
return files.Process(self.mdadm_command, '--zero_superblock',
device).success()


class RAID:
'''Класс для работы с RAID-массивами.'''
def __init__(self, mdadm_command):
self.mdadm_command = mdadm_command

def get_devices_info(self, raid_device):
'''Получить информацию о RAID-массиве.'''
device = udev.get_device_info(path=raid_device)
if udev.is_raid(device):
for raid_block in sysfs.glob(raid_device, 'md/dev-*', 'block'):
yield udev.get_device_info(sysfs.join_path(raid_block))

def get_devices(self, raid_device, path_name='DEVPATH'):
'''Получить устройства /dev, из которых состоит RAID-массив.
Не возвращает список этих устройств для раздела сделанного для RAID
(/dev/md0p1).'''
for device_info in self.get_devices_info(raid_device):
device_name = device_info.get(path_name, '')
if device_name:
yield device_name

def get_devices_syspath(self, raid_device):
'''Получить sysfs пути устройств, из которых состоит RAID-массив.'''
for device in self.get_devices(raid_device, path_name='DEVPATH'):
yield device

def remove_raid(self, raid_name):
'''Удалить RAID-массив.'''
raid_parts = list(self.get_devices(udev.get_syspath(name=raid_name)))
failed = False
failed |= not (self.mdadm_command.stop_raid(raid_name) or
self.mdadm_command.stop_raid(raid_name))
for device in raid_parts:
failed |= not (self.mdadm_command.zero_superblock(device) or
self.mdadm_command.zero_superblock(device))
return not failed


class DeviceFS(GenericFS):
'''Базовый класс для классов предназначенных для работы с sysfs и /dev'''
def __init__(self, filesystem=None):
if isinstance(filesystem, GenericFS):
self.filesystem = filesystem
else:
self.filesystem = files.RealFS('/')

def join_path(self, *paths):
output = os.path.join('/', *(_path[1:] if _path.startswith('/')
else _path for _path in paths))
return output

def exists(self, *paths):
return self.filesystem.exists(self.join_path(*paths))

def read(self, *paths):
return self.filesystem.read(self.join_path(*paths))

def realpath(self, *paths):
return self.filesystem.realpath(self.join_path(*paths))

def write(self, *args):
data = args[-1]
paths = args[:-1]
self.filesystem.write(self.join_path(*paths), data)

def listdir(self, *paths, full_path=False):
return self.filesystem.listdir(self.join_path(*paths),
full_path=full_path)

def glob(self, *paths):
for file_path in self.filesystem.glob(self.join_path(*paths)):
yield file_path


class SysFS(DeviceFS):
'''Класс для работы с sysfs.'''
BASE_DIRECTORY = '/sys'
PATH = {'firmware': 'firmware',
'efi': 'firmware/efi',
'efivars': 'firmware/efi/efivars',
'classnet': 'class/net',
'input': 'class/input',
'block': 'block',
'dmi': 'class/dmi/id',
'module': 'module',
'block_scheduler': 'queue/scheduler'}

def join_path(self, *paths):
output = os.path.join('/', *(_path[1:] if _path.startswith('/')
else _path for _path in paths))
if output.startswith(self.BASE_DIRECTORY):
return output
else:
return os.path.join(self.BASE_DIRECTORY, output[1:])

def glob(self, *paths):
for _path in self.filesystem.glob(self.join_path(*paths)):
yield _path[len(self.BASE_DIRECTORY):]


class DevFS(DeviceFS):
'''Класс для работы с /dev.'''
BASE_DIRECTORY = '/dev'

def join_path(self, *paths):
output = os.path.join('/', *(_path[1:] if _path.startswith('/')
else _path for _path in paths))
if output.startswith(self.BASE_DIRECTORY):
return output
else:
return os.path.join(self.BASE_DIRECTORY, output[1:])

def glob(self, *paths):
for _path in self.filesystem.glob(self.join_path(*paths)):
yield _path[len(self.BASE_DIRECTORY):]


class UdevAdmNull:
def info_property(self, path=None, name=None):
return {}

def info_export(self):
return ''

def settle(self, timeout=15):
pass

def trigger(self, subsystem=None):
pass

@property
def broken(self):
return False


class UdevAdmCommand(UdevAdmNull):
def __init__(self):
self.first_run = True

@property
def udevadm_cmd(self):
return files.get_program_path('/sbin/udevadm')

@property
def broken(self):
return not bool(self.udevadm_cmd)

def info_property(self, path=None, name=None):
if self.first_run:
self.trigger("block")
self.settle()
self.first_run = False

if name is not None:
type_query = "--name"
value = name
else:
type_query = "--path"
value = path
udevadm_output = files.Process(self.udevadm_cmd, "info",
"--query", "property",
type_query, value).read_lines()
output_items = []
for line in udevadm_output:
if '=' in line:
output_items.append(line.partition('=')[0::2])
return dict(output_items)

def info_export(self):
return files.Process(self.udevadm_cmd, 'info', '-e').read().strip()

def trigger(self, subsystem=None):
if subsystem:
files.Process(self.udevadm_cmd, 'trigger', '--subsystem-match',
subsystem).success()
else:
files.Process(self.udevadm_cmd, 'trigger').success()

def settle(self, timeout=15):
files.Process(self.udevadm_cmd, 'settle', '--timeout={}'.
format(timeout)).success()


class Udev:
'''Класс возвращающий преобразованную или кэшированную информацию о системе
из udev.'''
def __init__(self, udevadm=None):
self.path_cache = {}
self.name_cache = {}
if isinstance(udevadm, UdevAdmCommand):
self.udevadm = udevadm
else:
self.udevadm = UdevAdmCommand()
if self.udevadm.broken:
self.udevadm = UdevAdmNull()

def clear_cache(self):
self.path_cache = {}
self.name_cache = {}
self.udevadm = UdevAdmCommand()

def get_device_info(self, path=None, name=None):
if name is not None:
cache = self.name_cache
value = devfs.realpath(name)
name = value
else:
cache = self.path_cache
value = sysfs.realpath(path)
path = value

if value not in cache:
data = self.udevadm.info_property(path, name)
devname = data.get('DEVNAME', '')
devpath = data.get('DEVPATH', '')
if devname:
self.name_cache[devname] = data
if devpath:
devpath = '/sys{}'.format(devpath)
self.path_cache[devname] = data
return data
else:
return cache[value]

def get_block_devices(self):
for block in sysfs.glob(sysfs.PATH['block'], '*'):
yield block
blockname = os.path.basename(block)
for part in sysfs.glob(block, '{}*'.format(blockname)):
yield part

def syspath_to_devname(self, devices, drop_empty=True):
for device_path in devices:
info = self.get_device_info(path=device_path)
if 'DEVNAME' in info:
yield info['DEVNAME']
elif not drop_empty:
yield ''

def devname_to_syspath(self, devices, drop_empty=True):
for device_name in devices:
info = self.get_device_info(name=device_name)
if 'DEVPATH' in info:
yield info['DEVPATH']
elif not drop_empty:
yield ''

def get_devname(self, path=None, name=None, fallback=None):
if fallback is None:
if name:
fallback = name
else:
fallback = ''
return self.get_device_info(path, name).get('DEVNAME', fallback)

def get_syspath(self, path=None, name=None, fallback=None):
if fallback is None:
if path:
fallback = path
else:
fallback = ''
return self.get_device_info(path, name).get('DEVPATH', fallback)

def refresh(self, trigger_only=False):
if not trigger_only:
self.clear_cache()
files.quite_unlink('/etc/blkid.tab')
if not os.environ.get('EBUILD_PHASE'):
self.udevadm.trigger(subsystem='block')
self.udevadm.settle(15)

def is_cdrom(self, udev_data):
return udev_data.get('ID_CDROM', '') == '1'

def is_device(self, udev_data):
if 'DEVPATH' not in udev_data:
return False
return (sysfs.exists(udev_data.get('DEVPATH', ''), 'device') and
udev_data.get('DEVTYPE', '') == 'disk')

def is_raid(self, udev_data):
return (udev_data.get('MD_LEVEL', '').startswith('raid') and
udev_data.get('DEVTYPE', '') == 'disk')

def is_raid_partition(self, udev_data):
return (udev_data.get('MD_LEVEL', '').startswith('raid') and
udev_data.get('DEVTYPE', '') == 'partition')

def is_lvm(self, udev_data):
return 'DM_LV_NAME' in udev_data and 'DM_VG_NAME' in udev_data

def is_partition(self, udev_data):
return udev_data.get('DEVTYPE', '') == 'partition'

def is_block_device(self, udev_data):
return udev_data.get('SUBSYSTEM', '') == 'block'

def get_device_type(self, path=None, name=None):
info = self.get_device_info(path, name)
syspath = info.get('DEVPATH', '')
if self.is_cdrom(info):
return 'cdrom'
if self.is_device(info):
return 'disk'
if self.is_partition(info):
return '{}-partition'.format(
self.get_device_type(os.path.dirname(syspath)))
if self.is_raid(info):
for raid_device in raid.get_devices_syspath(syspath):
return '{}-{}'.format(self.get_device_type(raid_device),
info['MD_LEVEL'])
else:
return 'loop'
if self.is_lvm(info):
for lv_device in lvm.get_used_partitions(info['DM_VG_NAME'],
info['DM_LV_NAME']):
return '{}-lvm'.format(self.get_device_type(name=lv_device))
if self.is_block_device(info):
return 'loop'
return ''

def get_partition_type(self, path=None, name=None):
info = self.get_device_info(path, name)
if info.get('ID_PART_ENTRY_SCHEME') == 'dos':
part_id = info.get('ID_PART_ENTRY_TYPE', '')
part_number = info.get('ID_PART_ENTRY_NUMBER', '')
if part_id and part_number:
if part_id == '0x5':
return 'extended'
elif int(part_number) > 4:
return 'logical'
else:
return 'primary'
return info.get('ID_PART_TABLE_TYPE', '')

def _get_disk_devices(self, path=None, name=None):
info = udev.get_device_info(path=path, name=name)
syspath = info.get('DEVPATH', '')
if syspath:
# real device
if self.is_device(info):
yield True, info.get('DEVNAME', '')
# partition
elif self.is_partition(info):
for device in self._get_disk_devices(
path=os.path.dirname(syspath)):
yield device
# md raid
elif udev.is_raid(info):
yield False, info.get('DEVNAME', '')
for raid_device in sorted(raid.get_devices_syspath(syspath)):
for device in self._get_disk_devices(path=raid_device):
yield device
# lvm
elif udev.is_lvm(info):
yield False, info.get('DEVNAME', '')
for lv_device in lvm.get_used_partitions(info['DM_VG_NAME'],
info['DM_LV_NAME']):
for device in self._get_disk_devices(name=lv_device):
yield device

def get_disk_devices(self, path=None, name=None):
return sorted({
device for real_device, device in
self._get_disk_devices(path, name) if real_device
})

def get_all_base_devices(self, path=None, name=None):
try:
devices = (device for real_device, device in
self._get_disk_devices(path, name)
if not device.startswith('/dev/loop'))
if not self.is_partition(self.get_device_info(path, name)):
next(devices)
return list(unique(devices))
except StopIteration:
return []


sysfs = SysFS()
devfs = DevFS()

udev = Udev(UdevAdmCommand())
lvm = Lvm(LvmCommand())
raid = RAID(MdadmCommand())


if __name__ == '__main__':
print('FIND DEVICE BY PARTITION:')
print(find_device_by_partition('/dev/nvme0n1p4'))
print('LSPCI TEST:')
pprint(get_lspci_output())
print('GET PHYSICAL EXTENT SIZE:')
lvm_obj = Lvm(LvmCommand())
print(lvm_obj.get_physical_extent_size())
print('GET RUN COMMANDS:')
for command in files.get_run_commands(with_pid=True):
print(command)

+ 161
- 0
calculate/utils/io_module.py View File

@@ -0,0 +1,161 @@
import sys
import os


class ColorPrint:
def __init__(self, output=sys.stdout):
self.system_output = output

def get_console_width(self):
system_output_fd = self.system_output.fileno()
try:
width, hight = os.get_terminal_size(system_output_fd)
except OSError:
return 80

if width is None:
return 80
else:
return width

def print_right(self, left_offset, right_offset):
console_width = self.get_console_width()
number_of_spaces = console_width - left_offset - right_offset
self.system_output.write(' ' * number_of_spaces)

def print_colored(self, string, attribute='0', foreground='37',
background='40'):
print_parameters = []
if attribute:
print_parameters.append(attribute)
if foreground:
print_parameters.append(foreground)
if background:
print_parameters.append(background)
self.system_output.write('\033[{0}m{1}\033[0m'.
format(';'.join(print_parameters), string))

def print_bright_red(self, string):
self.print_colored(string,
attribute='1',
foreground='31')

def print_bright_green(self, string):
self.print_colored(string,
attribute='1',
foreground='32')

def print_bright_yellow(self, string):
self.print_colored(string,
attribute='1',
foreground='33')

def print_bright_blue(self, string):
self.print_colored(string,
attribute='1',
foreground='34')

def print_default(self, string):
try:
self.system_output.write(string)
except UnicodeError:
self.system_output.write(string.encode('utf-8'))
try:
self.system_output.flush()
except IOError:
exit(1)

def print_line(self, left_arg, right_arg, left_offset=0,
printBR=True):
COLORS = {'default': self.print_default,
'green': self.print_bright_green,
'red': self.print_bright_red,
'yellow': self.print_bright_yellow,
'blue': self.print_bright_blue}

for color, left_string in left_arg:
left_offset += len(left_string)
COLORS.get(color, self.print_default)(left_string)

right_offset = 0
for color, right_string in right_arg:
right_offset += len(right_string)
if right_offset:
self.print_right(left_offset, right_offset)
for color, right_string in right_arg:
COLORS.get(color, self.print_default)(right_string)
if printBR:
self.system_output.write('\n')
try:
self.system_output.flush()
except IOError:
exit(1)

def print_ok(self, string, left_offset=0, printBR=True):
self.system_output = sys.stdout
self.print_line((('green', ' * '), ('', string)),
(('blue', '['), ('green', ' ok '), ('blue', ']')),
left_offset=left_offset, printBR=printBR)

def print_only_ok(self, string, left_offset=0, printBR=True):
self.system_output = sys.stdout
self.print_line((('', string),),
(('blue', '['), ('green', ' ok '), ('blue', ']')),
left_offset=left_offset, printBR=printBR)

def print_not_ok(self, string, left_offset=0, printBR=True):
self.system_output = sys.stderr
self.print_line((('green', ' * '), ('', string)),
(('blue', '['), ('red', ' !! '), ('blue', ']')),
left_offset=left_offset, printBR=printBR)

def print_only_not_ok(self, string, left_offset=0, printBR=True):
self.system_output = sys.stderr
self.print_line((('', string),),
(('blue', '['), ('red', ' !! '), ('blue', ']')),
left_offset=left_offset, printBR=printBR)

def print_warning(self, string, left_offset=0, printBR=True):
self.system_output = sys.stdout
self.print_line((('yellow', ' * '), ('', string)),
(('', ''),),
left_offset=left_offset, printBR=printBR)

def print_error(self, string, left_offset=0, printBR=True):
self.system_output = sys.stderr
self.print_line((('red', ' * '), ('', string)),
(('', ''),),
left_offset=left_offset, printBR=printBR)

def print_success(self, string, left_offset=0, printBR=True):
self.system_output = sys.stdout
self.print_line((('green', ' * '), ('', string)),
(('', ''),),
left_offset=left_offset, printBR=printBR)

def print_info(self, string, left_offset=0, printBR=True):
self.system_output = sys.stdout
self.print_line((('blue', ' * '), ('', string)),
(('', ''),),
left_offset=left_offset, printBR=printBR)


# Заглушка вместо модуля вывода.
class IOModule:
def __init__(self):
self.console_output = ColorPrint()

def set_error(self, message):
self.console_output.print_error(message)

def set_warning(self, message):
self.console_output.print_warning(message)

def set_info(self, message):
self.console_output.print_info(message)

def set_success(self, message):
self.console_output.print_ok(message)

def set_failure(self, message):
self.console_output.print_not_ok(message)

+ 330
- 0
calculate/utils/mount.py View File

@@ -0,0 +1,330 @@
import os
import re
from . import device
from . import files
import xattr
import errno
from contextlib import contextmanager
from .tools import SingletonParam, flat_iterable
import tempfile


def is_mount(device_name):
'''Возвращает путь к точке монтирования, если устройство примонтировано
или '''
def find_names(old_device_name):
device_name = os.path.abspath(old_device_name)
yield device_name
if device_name.startswith('/dev'):
info = device.udev.get_device_info(name=device_name)
if 'DM_VG_NAME' in info and 'DM_LV_NAME' in info:
yield '/dev/mapper/{vg}-{lv}'.format(vg=info['DM_VG_NAME'],
lv=info['DM_LV_NAME'])

def get_overlay_mounts(line):
mounts = line.split(' ')
yield mounts[1]
for device_name in re.findall(
'(?:lowerdir=|upperdir=|workdir=)([^,]+)',
mounts[3]):
yield device_name

find_data = set(find_names(device_name))
output = ''
for mtab_line in files.read_file_lines('/etc/mtab'):
if 'overlay' not in mtab_line:
if ' ' in mtab_line:
mounts = set(mtab_line.split(' ')[:2])
if mounts & find_data:
output = (mounts - find_data).pop()
else:
mounts = set(get_overlay_mounts(mtab_line))
dest = mtab_line.split(' ')[1]
if mounts & find_data:
if dest in find_data:
output = 'overlay'
else:
return dest
return output


class MountHelperError(Exception):
pass


class MountHelperNotFound(Exception):
pass


class MountHelper:
'''Базовый класс для чтения файлов /etc/fstab и /etc/mtab.'''
DATA_FILE = '/etc/fstab'
NAME, DIR, TYPE, OPTS, FREQ, PASSNO = range(0, 6)

def __init__(self, data_file=None, devices=()):
from device import get_uuid_dict
if data_file:
self.DATA_FILE = data_file
self.cache = []
self.rotated_cache = []
self.uuid_dictionary = get_uuid_dict(devices=devices)
self.update_cache()

def _read_data(self):
with open(self.DATA_FILE, 'r') as data_file:
return data_file.read()

def update_cache(self):
def setlen(ar):
return ar[:6] + [''] * (6 - len(ar))

self.cache = []
for line in self._read_data().split('\n'):
line = line.strip()
if line and not line.startswith('#'):
separated_line = []
for part in line.split():
part = part.strip()
separated_line.append(part)
self.cache.append(separated_line)

for data in self.cache:
device_name = self.uuid_dictionary.get(data[0], data[0])

if device_name.startswith('/'):
device_name = os.path.realpath(device_name)

data[0] = device.udev.get_device_info(name=device_name).get(
'DEVNAME', data[0])
data[1] = data[1] if data[2] != 'swap' else 'swap'
self.rotated_cache = list(zip(*self.cache))

def get_from_fstab(self, what=DIR, where=NAME, is_in=None, is_equal=None,
contains=None, is_not_equal=None, all_entry=True):
if is_equal is not None:
def filter_function(tab_line):
return tab_line[where] == is_equal
elif is_in is not None:
def filter_function(tab_line):
return tab_line[where] in is_in
elif contains is not None:
def filter_function(tab_line):
return contains in tab_line[where]
else:
def filter_function(tab_line):
return tab_line[where] != is_not_equal
output = []
for line in filter(filter_function, self.cache):
output.append(line[what])
if all_entry:
return output
else:
return '' if not output else output[-1]

def get_fields(self, *fields):
fstab_fields = [self.rotated_cache[field] for field in fields]
return list(zip(*fstab_fields))

def is_read_only(self, what=DIR, is_equal=None):
tab_to_check = list(filter(lambda fstab_tab:
fstab_tab[what] == is_equal, self.cache))
if tab_to_check:
for data in tab_to_check:
options = data[self.OPTS].split(',')
if 'ro' in options:
return True
else:
return False
else:
raise MountHelperNotFound(is_equal)

def is_exist(self, what=DIR, is_equal=None, is_not_equal=None):
if is_equal is not None:
def filter_function(tab_line):
tab_line[what] == is_equal
else:
def filter_function(tab_line):
tab_line[what] != is_not_equal
return bool(filter(filter_function, self.cache))

@property
def writable(self, directory_path):
return not self.is_read_only(is_equal=directory_path)

@property
def read_only(self, directory_path):
return self.is_read_only(is_equal=directory_path)

@property
def exists(self, directory_path):
return self.is_exist(is_equal=directory_path)\
or self.is_exist(what=self.NAME,
is_equal=directory_path)


class FStab(MountHelper, metaclass=SingletonParam):
'''Класс для чтения содержимого /etc/fstab и его кеширования.'''
DATA_FILE = '/etc/fstab'


class Mounts(MountHelper):
'''Класс для чтения содержимого /etc/mtab и его кеширования.'''
DATA_FILE = '/etc/mtab'


class DiskSpaceError(Exception):
pass


class DiskSpace:
def __init__(self):
self.df_command = files.get_program_path('/bin/df')

def get_free(self, device=None, device_path=None):
if device:
mount_path = is_mount(device)
if not mount_path:
raise DiskSpaceError('Device {} must be mounted.'.
format(device))
device_path = device
df_process = files.Process(self.df_command, device_path, '-B1')
if df_process.success():
df_data = df_process.read().strip()
df_lines = df_data.split('\n')
if len(df_lines) >= 2:
columns = df_lines[1].split()
if len(columns) == 6:
return int(columns[3])
raise DiskSpaceError('Wrong df output:\n{}'.format(df_data))
else:
raise DiskSpaceError(str(df_process.read_error()))


def get_child_mounts(path_name):
'''Получить все точки монтирования, содержащиеся в пути.'''
mtab_file_path = '/etc/mtab'
if not os.access(mtab_file_path, os.R_OK):
return ''

with open(mtab_file_path) as mtab_file:
output = []
mtab = list(map(lambda line: line.split(' '), mtab_file))
mtab = [[line[0], line[1]] for line in mtab]

if path_name != 'none':
abs_path = os.path.abspath(path_name)
for tab_line in mtab:
if os.path.commonpath([abs_path, tab_line[1]]) == abs_path:
output.append(tab_line)
else:
abs_path = path_name
for tab_line in mtab:
if tab_line[0] == abs_path:
output.append(tab_line)
return output


class MountError(Exception):
pass


def mount(source, target, fstype=None, options=None):
parameters = [source, target]
if options is not None:
if isinstance(options, list) or isinstance(options, tuple):
options = ','.join(flat_iterable(options))
parameters.insert(0, '-o {}'.format(options))
if fstype is not None:
parameters.insert(0, '-t {}'.format(fstype))
mount_process = files.Process('/bin/mount', *parameters)
if mount_process.success():
return True
else:
raise MountError('Failed to mount {source} to {target}: {stderr}'
.format(source=source, target=target,
stderr=str(mount_process.read_error())))


def umount(target):
umount_process = files.Process('/bin/umount', target)
if umount_process.success():
return True
else:
raise MountError('Failed to umount {target}: {stderr}'
.format(target=target,
stderr=umount_process.read_error()))


class BtrfsError(Exception):
pass


class Btrfs:
check_path = None

def __init__(self, block_device):
self.block_device = block_device
if not os.path.exists(block_device):
raise BtrfsError('Device is not found.')

@contextmanager
def mount(self):
tempfile_path = None
try:
files.make_directory(self.check_path)
tempfile_path = tempfile.mkdtemp(prefix='btrfscheck-',
dir=self.check_path)
mount(self.block_device, tempfile_path, 'btrfs')
yield tempfile_path
except KeyboardInterrupt:
raise
except MountError as error:
if 'wrong fs type' in str(error):
raise BtrfsError('{} is not btrfs'.format(self.block))
else:
raise BtrfsError(str(error))
finally:
if tempfile_path:
if os.path.ismount(tempfile_path):
umount(tempfile_path)
os.rmdir(tempfile_path)

def get_compression(self, relative_path):
relative_path = relative_path.lstrip('/')
with self.mount() as device_path:
try:
absolute_path = os.path.join(device_path, relative_path)
if os.path.exists(absolute_path):
return xattr.get(absolute_path, 'btrfs.compression')
else:
return ''
except IOError as error:
if error.errno == errno.ENODATA:
return ''
raise BtrfsError('Failed to get btrfs compression.')

@property
def compression(self):
return self.get_compression('')

def set_compression(self, relative_path, value):
relative_path = relative_path.lstrip('/')
with self.mount() as device_path:
try:
absolute_path = os.path.join(device_path, relative_path)
if not os.path.exists(absolute_path):
files.make_directory(absolute_path)
return xattr.set(absolute_path, 'btrfs.compression', value)
except IOError as error:
if error.errno == errno.ENODATA:
return ''
raise BtrfsError('Failed to set btrfs compression.')

@compression.setter
def compression(self, value):
self.set_compression('', value)


if __name__ == '__main__':
print('GET CHILD MOUNTS TEST:')
print(get_child_mounts('/home'))

+ 150
- 0
calculate/utils/tools.py View File

@@ -0,0 +1,150 @@
import os
from functools import wraps
from abc import ABCMeta, abstractmethod
from inspect import getcallargs


class Cachable:
'''Базовый класс для создания классов, кэширующих вывод своих методов.
Декоратор @Cachable.method_cached() предназначен для указания методов
с кэшируемым выводом.'''
def __init__(self):
self.clear_method_cache()

def clear_method_cache(self):
self._method_cache = {}

@staticmethod
def method_cached(key=lambda *args, **kwargs: hash(args)):
def decorator(function):
function_name = function.__name__

@wraps(function)
def wrapper(self, *args, **kwargs):
keyval = key(*args, **kwargs)
assert isinstance(self, Cachable)
if function_name not in self._method_cache:
self._method_cache[function_name] = {}
cache = self._method_cache[function_name]
if keyval not in cache:
cache[keyval] = function(self, *args, **kwargs)
return cache[keyval]

@wraps(function)
def null_wrapper(self):
assert isinstance(self, Cachable)
if function_name not in self._method_cache:
self._method_cache[function_name] = function(self)
return self._method_cache[function_name]

if len(function.__code__.co_varnames) > 1:
return wrapper
else:
return null_wrapper

return decorator


class Singleton(type):
'''Метакласс для создания синглтонов.'''
_instances = {}

def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super().__call__(*args, **kwargs)
return cls._instances[cls]


class SingletonParam(type):
'''Метакласс для создания синглтонов по параметрам.'''
_instances = {}
_init = {}

def __init__(cls, class_name, base_classes, class_dict):
cls._init[cls] = class_dict.get('__init__', None)

def __call__(cls, *args, **kwargs):
init = cls._init[cls]
if init is not None:
key_value = (cls, frozenset(getcallargs(init, None, *args,
**kwargs).items()))
else:
key_value = cls

if key_value not in cls._instances:
cls._instances[key_value] = super().__call__(*args, **kwargs)
return cls._instances[key_value]


class GenericFS(metaclass=ABCMeta):
'''Абстрактный класс для работы с файловыми системами.'''
@abstractmethod
def exists(self, path):
pass

@abstractmethod
def read(self, path):
pass

@abstractmethod
def glob(self, path):
pass

@abstractmethod
def realpath(self, path):
pass

@abstractmethod
def write(self, path, data):
pass

@abstractmethod
def listdir(self, path, fullpath=False):
pass


def get_traceback_caller(exception_type, exception_object,
exception_traceback):
'''Возвращает имя модуля, в котором было сгенерировано исключение,
и соответствующий номер строки.'''
while exception_traceback.tb_next:
exception_traceback = exception_traceback.tb_next

line_number = exception_traceback.tb_lineno

module_path, module_name = os.path.split(exception_traceback.tb_frame.
f_code.co_filename)
if module_name.endswith('.py'):
module_name = module_name[:-3]
full_module_name = [module_name]
while (module_path and module_path != '/' and not
module_path.endswith('site-packages')):
module_path, package_name = os.path.split(module_path)
full_module_name.insert(0, package_name)
if module_path.endswith('site-packages'):
module_name = '.'.join(full_module_name)
return module_name, line_number


def unique(iterable):
'''Возвращает итерируемый объект, содержащий только уникальные элементы
входного объекта, сохраняя их порядок.
'''
output = []
for element in iterable:
if element not in output:
output.append(element)
return output


def flat_iterable(iterable, types=(list, tuple, map, zip, filter)):
'''Распаковывает все вложенные итерируемые объекты во входном объекте.
Например:
[1, 2, [3, 4, [5, 6], 7], [8, 9], 10] -> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
'''
if isinstance(iterable, types):
for it in iterable:
for sub_iterable in flat_iterable(it, types=types):
yield sub_iterable
else:
yield iterable

+ 0
- 0
calculate/vars/os/__init__.py View File


+ 0
- 0
tests/__init__.py View File


+ 0
- 0
tests/templates/format/testfiles/__init__.py View File


+ 52
- 0
tests/templates/test_directory_processor.py View File

@@ -0,0 +1,52 @@
import pytest
from calculate.templates.template_processor import DirectoryProcessor
from calculate.templates.template_engine import Variables


# Вместо модуля переменных.
group = Variables({'bool': True,
'list': [1, 2, 3]})

variables = Variables({'variable_1': 'value_1',
'variable_2': 'value_2',
'group': group})

install = Variables({'os_disk_dev': 'os_disk_dev_value',
'version': 1.5,
'number': 128,
'boolean': False,
'type': 'static',
'path': '/usr/sbin'})

merge = Variables({'var_1': 674,
'var_2': 48,
'version': 1.0,
'calculate_domains': 'lists.calculate-linux.org',
'ip_value': '127.0.0.0/8'})

cl_template = Variables({
'path':
'tests/templates/testfiles/test_dir_1,tests/templates/testfiles/test_dir_2'
})

cl_chroot = Variables({'path': '/etc'})

main = Variables({'cl_template': cl_template,
'cl_chroot': cl_chroot})

datavars = Variables({'install': install,
'merge': merge,
'variables': variables,
'main': main,
'custom': Variables()})


@pytest.mark.directory_processor
class TestDirectoryProcessor:
def test_just_for_debug(self):
try:
dir_processor = DirectoryProcessor('install', datavars_module=datavars,
package='package_1')
dir_processor.process_template_directories()
except Exception as error:
pytest.fail('Unexpected exception: {}'.format(str(error)))