Browse Source

Добавлен парсер Emerge и прочие вспомогательные файлы

master3.3 3.2.0_alpha2
Mike khiretskiy 8 years ago
parent
commit
5c8b5438fc
  1. 24
      setup.py
  2. 603
      update/emerge_parser.py
  3. 450
      update/package_tools.py
  4. 323
      update/update.py
  5. 297
      update/utils/cl_update.py
  6. 21
      update/variables/update.py
  7. 3
      update/wsdl_update.py

24
setup.py

@ -20,21 +20,21 @@
__app__ = "calculate-update"
__version__ = "3.1.8"
import os
from distutils.core import setup
from calculate.install_data import install_data
data_files = [('/usr/libexec/calculate',[('data/cl-git-wrapper',0755)])]
data_files = [('/usr/libexec/calculate', [('data/cl-git-wrapper', 0755)])]
setup(
name = __app__,
version = __version__,
description = "Update system utilities",
author = "Calculate Ltd.",
author_email = "support@calculate.ru",
url = "http://calculate-linux.org",
license = "http://www.apache.org/licenses/LICENSE-2.0",
package_dir = {'calculate.update': "update"},
data_files = data_files,
packages = ['calculate.update','calculate.update.utils','calculate.update.variables'],
name=__app__,
version=__version__,
description="Update system utilities",
author="Calculate Ltd.",
author_email="support@calculate.ru",
url="http://calculate-linux.org",
license="http://www.apache.org/licenses/LICENSE-2.0",
package_dir={'calculate.update': "update"},
data_files=data_files,
packages=['calculate.update', 'calculate.update.utils',
'calculate.update.variables'],
cmdclass={'install_data': install_data})

603
update/emerge_parser.py

@ -0,0 +1,603 @@
#-*- coding: utf-8 -*-
# Copyright 2014 Calculate Ltd. http://www.calculate-linux.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from os import path
import re
import sys
from calculate.lib.utils.colortext.palette import TextState
from calculate.lib.utils.tools import ignore
from package_tools import EmergePackage, PackageList, EmergeUpdateInfo, \
UnmergePackage, EmergeRemoveInfo, Git, GitError
Colors = TextState.Colors
import pexpect
from calculate.lib.utils.files import getProgPath, readLinesFile, listDirectory, \
writeFile, readFile
from calculate.lib.utils.colortext.output import XmlOutput
from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter,
XmlConverter)
from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _
setLocalTranslate('cl_update3', sys.modules[__name__])
__ = getLazyLocalTranslate(_)
class EmergeError(Exception):
"""
Ошибка при сборке пакетов
"""
class EmergeNeedRootError(EmergeError):
pass
class CommandExecutor(object):
"""
Запуск программы для объекта Emerge
"""
logfile = '/var/log/calculate/lastcommand.log'
def __init__(self, cmd, params, env=None, cwd=None, logfile=None):
self.cwd = cwd
self.env = env
self.cmd = cmd
self.params = params
self.child = None
if logfile:
self.logfile = logfile
def execute(self):
if self.child is None:
self.child = pexpect.spawn(self.cmd,
logfile=open(self.logfile, 'w'),
env=self.env, cwd=self.cwd, timeout=None)
return self.child
def close(self):
if self.child is not None:
self.child.close()
self.child = None
def success(self):
if self.child:
if self.child.isalive():
self.child.wait()
return self.child.exitstatus == 0
return False
def failed(self):
return not self.success()
def send(self, s):
if self.child:
self.child.send(s)
class EmergeCommand(CommandExecutor):
"""
Запуск emerge для последующего анализирования
"""
# параметры по умолчанию
default_params = ["-av", "--color=y", "--nospinner"]
cmd = getProgPath("/usr/bin/emerge")
def __init__(self, packages, extra_params=None, env=None, cwd=None,
logfile=None):
extra_params = extra_params or []
self.child = None
self.packages = packages
self.params = self.default_params + extra_params
default_env = {'CLEAN_DELAY': '0'}
default_env.update(os.environ)
self.env = env or default_env
self.cwd = cwd
if logfile:
self.logfile = logfile
def execute(self):
if self.child is None:
self.child = pexpect.spawn(self.cmd, self.params + self.packages,
logfile=open(self.logfile, 'w'),
env=self.env, cwd=self.cwd, timeout=None)
return self.child
class EmergeInformationBlock(object):
_color_block = "(?:\033\[[^m]+?m)?"
_new_line = "\r*\n"
token = None
end_token = ["\n"]
re_block = None
action = None
re_match_type = type(re.match("", ""))
re_type = type(re.compile(""))
def __init__(self, parent):
"""
:type parent: EmergeParser
"""
self.result = None
self.text_converter = parent.text_converter
self.parent = parent
self.parent.add_element(self)
self.children = []
def add_element(self, element):
self.children.append(element)
def __str__(self):
if type(self.result) == self.re_match_type:
return self.result.group()
else:
return self.result or ""
def __nonzero__(self):
return bool(self.result)
def __len__(self):
if self.result is None:
return 0
else:
return len(self.result)
def __contains__(self, item):
if self.result is None:
return 0
else:
return item in self.result
def _get_text(self, result):
"""
Получить результат из регулярки и преобразовать его через self.converter
"""
if result:
return self.text_converter.transform(result.rstrip())
return ""
def get_block(self, child):
try:
token = child.match
if type(self.end_token) == self.re_type:
child.expect(self.end_token)
match = child.match.group()
else:
child.expect_exact(self.end_token)
match = child.match
self.get_data(self.re_block.search(
token + child.before + match))
except pexpect.EOF:
child.buffer = "".join(
[x for x in (child.before, child.after, child.buffer)
if type(x) == str])
def get_data(self, match):
self.result = self._get_text(match.group(1))
class InstallPackagesBlock(EmergeInformationBlock):
"""
Блок emerge содержащий список пакетов для установки
"""
list = PackageList([])
remove_list = PackageList([])
_new_line = EmergeInformationBlock._new_line
token = "\n["
end_token = ["\r\n\r", "\n\n"]
re_block = re.compile(r"((?:^\[.*?{nl})+)".format(nl=_new_line),
re.MULTILINE)
def get_data(self, match):
super(InstallPackagesBlock, self).get_data(match)
list_block = XmlConverter().transform(self.result).split('\n')
self.list = PackageList(map(EmergeUpdateInfo, list_block))
self.remove_list = PackageList(map(EmergeRemoveInfo, list_block))
class UninstallPackagesBlock(EmergeInformationBlock):
"""
Блок emerge содержащий список удаляемых пакетов
"""
list = PackageList([])
_new_line = EmergeInformationBlock._new_line
_color_block = EmergeInformationBlock._color_block
token = ["These are the packages that would be unmerged",
"Calculating removal order"]
end_token = re.compile("All selected packages:.*\n")
re_block = re.compile(r"All selected packages: (.*?){nl}".
format(nl=_new_line, c=_color_block), re.DOTALL)
def get_data(self, match):
super(UninstallPackagesBlock, self).get_data(match)
list_block = XmlConverter().transform(self.result).split()
self.list = PackageList(map(EmergePackage, list_block))
class FinishEmergeGroup(EmergeInformationBlock):
"""
Блок завершения команды
"""
token = pexpect.EOF
# регуляреное выражение, определяющее содержит ли блок
# сообщения об ошибках
re_failed = re.compile(
r"Fetch instructions for \S+:|"
r"The following.*are necessary to proceed|"
r"!!! Multiple package .* slot have been pulled|"
r"no ebuilds to satisfy|"
r"Dependencies could not be completely resolved due to",
re.MULTILINE)
def get_block(self, child):
if child.isalive():
child.wait()
if child.exitstatus != 0 or self.re_failed.search(child.before):
self.children_get_block(child)
else:
self.result = True
def children_get_block(self, child):
for block in self.children:
block.get_block(child)
def children_action(self, child):
for block in (x for x in self.children if x.result and x.action):
if block.action(child) is False:
break
def action(self, child):
self.children_action(child)
return False
class PrepareErrorBlock(EmergeInformationBlock):
"""
Блок информации с ошибками при получении списка устанавливаемых пакетов
"""
token = None
re_drop = re.compile("news items need reading|"
"Use eselect news|"
"Calculating dependencies|"
"to read news items|"
"Local copy of remote index is up-to-date|"
"These are the packages that would be merged|"
"Process finished with exit code")
re_multi_empty_line = re.compile("(?:<br/>){3,}", re.DOTALL)
re_strip_br = re.compile("^(?:<br/>)+|(?:<br/>)+$", re.DOTALL)
def remove_needless_data(self, data):
return "\n".join([x for x in data.split('\n')
if not self.re_drop.search(x)])
def strip_br(self, data):
return self.re_strip_br.sub(
"",
self.re_multi_empty_line.sub("<br/><br/>", data))
def get_block(self, child):
self.result = self.strip_br(
self._get_text(self.remove_needless_data(child.before)))
def action(self, child):
raise EmergeError(_("Emerge failed"))
class DownloadSizeBlock(EmergeInformationBlock):
"""
Размер скачиваемых обновлений
"""
token = "Size of downloads:"
re_block = re.compile(r"Size of downloads:\s(\S+\s\S+)")
def __str__(self):
if self.result:
return self.result
else:
return "0 kB"
class QuestionBlock(EmergeInformationBlock):
"""
Блок вопроса
"""
default_answer = "no"
_color_block = EmergeInformationBlock._color_block
token = "Would you like"
end_token = ["]", "\n"]
re_block = re.compile(
"(Would you.*)\[{c}Yes{c}/{c}No{c}".format(c=_color_block))
def action(self, child):
if self.result:
child.send("%s\n" % self.default_answer)
class NeedRootBlock(EmergeInformationBlock):
"""
Пользователь не явеляется root
"""
token = "This action requires superuser access"
def get_data(self, child):
self.result = True
def action(self, child):
raise EmergeNeedRootError(_("This action requires superuser access"))
class NotifierInformationBlock(EmergeInformationBlock):
"""
Информационный блок поддерживающий observing
"""
def __init__(self, parent):
super(NotifierInformationBlock, self).__init__(parent)
self.observers = []
def get_data(self, match):
self.result = match
def add_observer(self, f):
self.observers.append(f)
def notify(self, observer, groups):
observer(groups)
def action(self, child):
if self.result and self.observers:
groups = self.result.groups()
for observer in self.observers:
self.notify(observer, groups)
class EmergingPackage(NotifierInformationBlock):
"""
Запуск устанавливаемого пакета
ObserverFunc: (package, num=number, max_num=number, binary=binary)
"""
_color_block = EmergeInformationBlock._color_block
token = ">>> Emerging "
re_block = re.compile(
"Emerging (binary )?\({c}(\d+){c} "
"of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
def notify(self, observer, groups):
observer(EmergePackage(groups[3]), num=groups[1], max_num=groups[2],
binary=bool(groups[0]))
class UnemergingPackage(NotifierInformationBlock):
"""
Запуск устанавливаемого пакета
ObserverFunc: (package, num=number, max_num=number)
"""
_color_block = EmergeInformationBlock._color_block
token = ">>> Unmerging"
re_block = re.compile(
r"Unmerging (?:\({c}(\d+){c} "
r"of {c}(\d+){c}\) )?(\S+)\.\.\.".format(c=_color_block))
def notify(self, observer, groups):
observer(EmergePackage(groups[2]), num=groups[0], max_num=groups[1])
class InstallingPackage(NotifierInformationBlock):
"""
Запуск устанавливаемого пакета
ObserverFunc: (package, binary=binary)
"""
_color_block = EmergeInformationBlock._color_block
binary = None
token = ">>> Installing "
re_block = re.compile(
"Installing \({c}(\d+){c} "
"of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
def notify(self, observer, groups):
binary = bool(self.binary and groups[2] in self.binary)
observer(EmergePackage(groups[2]), binary=binary)
def mark_binary(self, package):
if self.binary is None:
self.binary = []
self.binary.append(str(package))
class EmergeingErrorBlock(EmergeInformationBlock):
"""
Блок содержит информацию об ошибке во время сборки пакета
"""
token = ["* ERROR: ", " * \033[39;49;00mERROR: "]
end_token = "Working directory:"
re_block = re.compile("ERROR: (\S*) failed \([^)]+\).*?"
"The complete build log is located at '([^']+)",
re.DOTALL)
package = ""
def get_data(self, match):
self.result = self._get_text(match.group(2).rstrip())
self.package = match.group(1)
@property
def log(self):
return self.text_converter.transform(readFile(self.result))
def action(self, child):
raise EmergeError(_("Emerge %s is failed") % self.package)
class EmergeParser(object):
"""
Парсер вывода emerge
"""
def __init__(self, command, run=False):
self.text_converter = ConsoleCodes256Converter(XmlOutput())
self.command = command
self.elements = {}
self.install_packages = InstallPackagesBlock(self)
self.uninstall_packages = UninstallPackagesBlock(self)
self.question = QuestionBlock(self)
self.finish_block = FinishEmergeGroup(self)
self.need_root = NeedRootBlock(self)
self.prepare_error = PrepareErrorBlock(self.finish_block)
self.download_size = DownloadSizeBlock(self)
self.emerging_error = EmergeingErrorBlock(self)
self.installing = InstallingPackage(self)
self.uninstalling = UnemergingPackage(self)
self.emerging = EmergingPackage(self)
self.emerging.add_observer(self.mark_binary)
if run:
self.run()
def mark_binary(self, package, binary=False, **kw):
if binary:
self.installing.mark_binary(package)
def add_element(self, element):
if element.token:
if type(element.token) == list:
for token in element.token:
self.elements[token] = element
else:
self.elements[element.token] = element
def run(self):
"""
Запустить команду
"""
child = self.command.execute()
while True:
index = child.expect_exact(self.elements.keys())
element = self.elements.values()[index]
element.get_block(child)
if element.action:
if element.action(child) is False:
break
def close(self):
self.command.close()
def __enter__(self):
return self
def __exit__(self, *exc_info):
self.close()
class MtimeCheckvalue(object):
def __init__(self, *fname):
self.fname = fname
def value_func(self, fn):
return str(int(os.stat(fn).st_mtime))
def get_check_values(self, file_list):
for fn in file_list:
if path.exists(fn) and not path.isdir(fn):
yield fn, self.value_func(fn)
else:
for k, v in self.get_check_values(
listDirectory(fn, fullPath=True)):
yield k, v
def checkvalues(self):
return self.get_check_values(self.fname)
class GitCheckvalue(object):
def __init__(self, rpath):
self.rpath = rpath
self.git = Git()
def checkvalues(self):
with ignore(GitError):
if self.git.is_git(self.rpath):
yield self.rpath, Git().getCurrentCommit(self.rpath)
class EmergeCache(object):
"""
Кэш пакетов
"""
cache_file = '/var/lib/calculate/calculate-update/world.cache'
# список файлов проверяемый по mtime на изменения
check_list = [MtimeCheckvalue('/etc/make.conf',
'/etc/portage',
'/etc/make.profile',
'/var/cache/edb/binhost')]
def __init__(self):
self.files_control_values = {}
self.pkg_list = PackageList([])
def set_cache(self, package_list):
"""
Установить в кэш список пакетов
"""
with writeFile(self.cache_file) as f:
for fn, val in self.get_control_values().items():
f.write("{fn}={val}\n".format(fn=fn, val=val))
f.write('\n')
for pkg in package_list:
f.write("%s\n"% str(pkg))
def drop_cache(self):
if path.exists(self.cache_file):
with ignore(OSError):
os.unlink(self.cache_file)
def get_cached_package_list(self):
self.read_cache()
if self.check_actuality():
return self.pkg_list
return None
def check_actuality(self):
"""
Кэш считается актуальным если ни один из файлов не менялся
"""
return self.get_control_values() == self.files_control_values
def read_cache(self):
self.files_control_values = {}
cache_file_lines = readLinesFile(self.cache_file)
for line in cache_file_lines:
if not "=" in line:
break
k, v = line.split('=')
self.files_control_values[k] = v.strip()
self.pkg_list = PackageList(cache_file_lines)
def get_control_values(self):
def generate():
for obj in self.check_list:
for checkvalue in obj.checkvalues():
yield checkvalue
return dict(generate())

450
update/package_tools.py

@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import Mapping
from collections import Mapping, defaultdict
import re
import sys
@ -28,12 +28,10 @@ from os import path
from calculate.lib.utils.files import (getProgPath, STDOUT,
PercentProgress, process, readFile,
readLinesFile)
from calculate.lib.utils.colortext.output import XmlOutput
from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter,
XmlConverter)
from calculate.lib.utils.common import cmpVersion
from contextlib import closing
import xml.etree.ElementTree as ET
from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
@ -127,7 +125,6 @@ class Layman:
self._add_to_makeconf(rpath)
return True
class Git:
"""
Объект для управление git репозиторием
@ -487,7 +484,7 @@ class EmergePackage(Mapping):
pv = r"(?:-(\d[^-]*?))?"
pr = r"(?:-(r\d+))?"
tbz = r"(?:.(tbz2))?"
slot = r'(?::(\d+(?:\.\d+)*(?:/\d+(?:\.\d+))?))?'
slot = r'(?::(\w+(?:\.\w+)*(?:/\w+(?:\.\w+)*)?))?'
repo = r'(?:::(\w+))?'
reParse = re.compile(
@ -515,7 +512,10 @@ class EmergePackage(Mapping):
'REPO': x[REPO] or self.default_repo,
'CATEGORY/PN': "%s/%s" % (x[CATEGORY], x[PN]),
'PR': x[PR] or 'r0'}
d['PVR'] = "%s-%s" % (d['PV'], d['PR'])
if x[PR]:
d['PVR'] = "%s-%s" % (d['PV'], d['PR'])
else:
d['PVR'] = d['PV']
if d['PF'].endswith('-r0'):
d['PF'] = d['PF'][:-3]
return d.copy()
@ -538,27 +538,31 @@ class EmergePackage(Mapping):
if "CATEGORY/PN" in version and "PVR" in version:
if self['CATEGORY/PN'] < version['CATEGORY/PN']:
return True
version = version['PVR']
return cmpVersion(self['PVR'], version) == -1
elif self['CATEGORY/PN'] > version['CATEGORY/PN']:
return False
version = "%s-%s" % (version['PV'], version['PR'])
currentVersion = "%s-%s" % (self['PV'], self['PR'])
return cmpVersion(currentVersion, version) == -1
def __eq__(self, version):
if "CATEGORY" in version and "PF" in version:
return ("%s/%s" % (self['CATEGORY'], self['PF']) ==
"%s/%s" % (version['CATEGORY'], version['PF']))
else:
return cmpVersion(self['PVR'], version) == 0
currentVersion = "%s-%s" % (self['PV'], self['PR'])
return cmpVersion(currentVersion, version) == 0
def __init__(self, package):
if isinstance(package, EmergePackage):
self.__result = package.__result
self.__package = package.__package
self._package = package._package
else:
self.__package = package
self._package = package
self.__result = None
def __getitem__(self, item):
if not self.__result:
self.__result = self._parsePackageString(self.__package)
self.__result = self._parsePackageString(self._package)
return self.__result[item]
def __repr__(self):
@ -568,6 +572,85 @@ class EmergePackage(Mapping):
return "%s/%s" % (self['CATEGORY'], self['PF'])
class PackageInformation:
"""
Объект позволяет получать информацию о пакете из eix
"""
eix_cmd = getProgPath("/usr/bin/eix")
query_packages = []
information_cache = defaultdict(dict)
fields = ["DESCRIPTION"]
def __init__(self, pkg):
self._pkg = pkg
if not pkg in self.query_packages:
self.query_packages.append(pkg)
def __getitem__(self, item):
if not self._pkg['CATEGORY/PN'] in self.information_cache:
self.query_information()
try:
return self.information_cache[self._pkg['CATEGORY/PN']][item]
except KeyError:
return ""
def query_information(self):
pkg_list = "|".join(
[x['CATEGORY/PN'].replace("+", r"\+") for x in self.query_packages])
output = pexpect.spawn(self.eix_cmd, ["--xml", pkg_list]).read()
xml = ET.fromstring(output)
for pkg in self.query_packages:
cat_pn = pkg['CATEGORY/PN']
if not cat_pn in self.information_cache:
descr_node = xml.find(
'category[@name="%s"]/package[@name="%s"]/description'
% (pkg['CATEGORY'], pkg['PN']))
if descr_node is not None:
self.information_cache[cat_pn]['DESCRIPTION'] = \
descr_node.text
while self.query_packages:
self.query_packages.pop()
@classmethod
def add_info(cls, pkg):
pkg.info = cls(pkg)
return pkg
class UnmergePackage(EmergePackage):
"""
Информация об обновлении одного пакета
"""
re_pkg_info = re.compile("^\s(\S+)\n\s+selected:\s(\S+)",re.MULTILINE)
def __init__(self, package):
super(UnmergePackage, self).__init__(package)
if not isinstance(package, EmergePackage):
self._package = self.convert_package_info(package)
def convert_package_info(self, package):
match = self.re_pkg_info.search(package)
if match:
return "%s-%s" % match.groups()
return ""
def recalculate_update_info(cls):
"""
Добавить
"""
cls.update_info = re.compile(
r"^({install_info})\s+({atom_info})\s*(?:{prev_version})?"
r"\s*({use_info})?.*?({pkg_size})?$".format(
install_info=cls.install_info,
atom_info=cls.atom_info,
prev_version=cls.prev_version,
use_info=cls.use_info,
pkg_size=cls.pkg_size), re.MULTILINE)
return cls
@recalculate_update_info
@total_ordering
class EmergeUpdateInfo(Mapping):
"""
@ -578,27 +661,22 @@ class EmergeUpdateInfo(Mapping):
atom_info = r"\S+"
use_info = 'USE="[^"]+"'
prev_version = "\[([^\]]+)\]"
update_info = re.compile(
r"^({install_info})\s+({atom_info})\s+(?:{prev_version})?"
r"\s*({use_info})?".format(install_info=install_info,
atom_info=atom_info,
prev_version=prev_version,
use_info=use_info))
pkg_size = r"\d+ \w+"
attrs = ['binary', 'REPLACING_VERSIONS']
attrs = ['binary', 'REPLACING_VERSIONS', 'SIZE']
def __init__(self, data):
self._data = data
self._package = None
self._info = {}
def _parseData(self):
r = self.update_info.search(self._data)
if r:
self._info['binary'] = r.group(2) == 'binary'
self._package = EmergePackage(r.group(3))
self._info['REPLACING_VERSIONS'] = r.group(4) or ""
self._info['SIZE'] = r.group(6) or ""
def __iter__(self):
return chain(EmergePackage.attrs, self.attrs)
@ -639,16 +717,12 @@ class EmergeUpdateInfo(Mapping):
def __str__(self):
return "%s/%s" % (self['CATEGORY'], self['PF'])
class EmergeError(Exception):
@recalculate_update_info
class EmergeRemoveInfo(EmergeUpdateInfo):
"""
Ошибка при сборке пакетов
Информация об удалении одного пакета (в списке обновляемых пакетов)
"""
(NEED_ROOT, NO_EBUILD, CONFLICT, CUSTOM) = range(0, 4)
def __init__(self, msg, errno=CUSTOM):
Exception.__init__(self, msg)
self.errno = errno
install_info = "\[(uninstall)[^\]]+\]"
class Eix:
@ -721,277 +795,7 @@ class Eix:
return iter(())
class Emerge:
"""
Объект для выполнения сборки пакетов
"""
# параметры по умолчанию
default_params = ["-av", "--color=y", "--nospinner"]
cmd = getProgPath("/usr/bin/emerge")
# шаблон выборки списка пакетов
install_pkgs_pattern = \
"(\\[[^\\]]+\\]\\s*(\\S+).*(?:{nl}|$))".format(nl="\r*\n")
re_install_pkgs = re.compile("^%s+" % install_pkgs_pattern,
re.MULTILINE)
# конфликтный пакет
_conflict_pkg = r"[\w-]+/[\w-]+:\d+"
# конкретная версия конфликтного пакета
_conflict_candidate = r" \(.*"
# пакеты в которым необходим конфликтный пакет
_conflict_req = r" \S.*"
# шаблон конфликтного блока
conflict_block = \
"({nl}{nl}({0}{nl}({1}{nl})+{nl})+)".format(_conflict_candidate,
_conflict_req,
nl="\r*\n")
re_conflict_block = re.compile(
"^{0}{conflict_block}([\w\W]+{conflict_block})*".format(
_conflict_pkg, conflict_block=conflict_block),
re.MULTILINE)
# шаблон устновки консольного цвета
_color_block = "(?:\033\[[^m]+?m)?"
re_custom_error = re.compile(
r"^Calculating dependencies.*{nl}".format(nl="\r*\n") +
"%s*" % install_pkgs_pattern +
r"(?P<result>[\w\W]+)", re.MULTILINE)
# regexp для удаления неиспользуемой информации
re_cut_blocks = \
re.compile(r"(It may be possible to solve.*?"
"solve this conflict automatically\.|"
"For more information, see.*?Gentoo Handbook\.)\s*", re.S)
expect_need_root = "This action requires superuser access"
expect_conflict = "Multiple package[ \w]+pulled"
expect_to_merge = "Would you like to merge"
expect_nothing_merge = ("Nothing to merge|No outdated packages|"
"\s0 packages")
expect_no_ebuild = "no ebuilds to satisfy \""
expect_custom_question = "{c}Yes{c}/{c}No{c}".format(c=_color_block)
expect_emerge_package = ("Emerging (binary )?\({c}(\d+){c} "
"of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(
c=_color_block))
expect_install_package = ("Installing (binary )?\({c}(\d+){c} "
"of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(
c=_color_block))
expect_error_on_install = "ERROR: \S* failed \([^)]+\)"
def __init__(self, packages, extra_params=[], env=None, cwd=None,
command_runner=None,
converter=ConsoleCodes256Converter(XmlOutput())):
self.text_converter = converter
self.packages = packages
self.params = self.default_params + extra_params
self.child = None
self.update_block = None
self.conflict_block = None
self.update_packages = None
self.custom_error = ""
self.env = env
self.cwd = cwd
# событие начало сборки
self.event_emerging = []
# событие начало установки
self.event_installing = []
self.error_log = None
def handle_emerging(self, f):
"""
Добавить обработку начала сборки пакета
"""
if not f in self.event_emerging:
self.event_emerging.append(f)
def handle_installing(self, f):
"""
Добавить обработку начала установки пакета
"""
if not f in self.event_installing:
self.event_installing.append(f)
def _get_text(self, re_result, already_text=False):
"""
Получить результат из регулярки и преобразовать его через self.converter
"""
if re_result:
if not already_text:
re_result = re_result.group()
result = self.re_cut_blocks.sub("", re_result)
return self.text_converter.transform(result.strip())
return ""
def _get_custom_error_output(self, buf):
"""
Получить вывод об прочих ошибках в ходе вычисления зависимостей
"""
r = self.re_custom_error.search(buf)
if r:
return self._get_text(r.groupdict()["result"], already_text=True)
return ""
def _get_package_list_output(self, buf):
"""
Получить из вывода emerge часть текста со списком пакетов
"""
return self._get_text(self.re_install_pkgs.search(buf))
def _get_conflict_list_output(self, buf):
"""
Разобрать вывод о конфликтах среди пакетов
"""
return self._get_text(self.re_conflict_block.search(buf))
def get_update_block(self):
"""
Получить список устанавливаемых пакетов
"""
if self.update_block is None:
self.execute()
events = {v: k for k, v in enumerate(sorted([
self.expect_need_root,
self.expect_conflict,
self.expect_to_merge,
self.expect_nothing_merge,
self.expect_no_ebuild,
self.expect_custom_question,
pexpect.EOF]))}
res = self.child.expect(sorted(events.keys()))
if res == events[self.expect_need_root]:
self.process_need_root()
elif res == events[self.expect_to_merge]:
self.update_block = (
self._get_package_list_output(self.child.before))
elif res == events[self.expect_nothing_merge]:
self.update_block = ""
elif res == events[self.expect_no_ebuild]:
raise EmergeError(_("No ebuilds to satisfy %s") %
self.child.buffer.split('"', 1)[0],
errno=EmergeError.NO_EBUILD)
elif res == events[self.expect_conflict]:
self.update_block = \
self._get_package_list_output(self.child.before)
self.conflict_block = \
self._get_conflict_list_output(self.child.read())
raise EmergeError(_("Multiple package instances within "
"a single package slot"),
errno=EmergeError.CONFLICT)
elif res == events[self.expect_custom_question]:
self.custom_error = self._get_custom_error_output(
self.child.before)
self.child.sendline("no")
raise EmergeError(_("Unexpected question"))
elif res == events[pexpect.EOF]:
if re.search("Total:\s+\d+ package", self.child.before):
self.update_block = (
self._get_package_list_output(self.child.before))
else:
self.custom_error = self._get_custom_error_output(
self.child.before)
raise EmergeError(_("Failed to emerge"))
return self.update_block
def notifyEmergingPackage(self, package, num=1, max_num=2, binary=False):
[f(EmergePackage(package), num=num, max_num=max_num, binary=binary)
for f in self.event_emerging]
def notifyInstallingPackage(self, package):
[f(EmergePackage(package)) for f in self.event_installing]
def install(self):
"""
Install packages
"""
self.execute()
self.child.sendline("yes")
pkg_name = ""
events = {v: k for k, v in enumerate(sorted([
self.expect_need_root,
self.expect_emerge_package,
self.expect_install_package,
self.expect_error_on_install,
pexpect.EOF]))}
while True:
res = self.child.expect(sorted(events.keys()))
if res == events[self.expect_need_root]:
self.process_need_root()
elif res in (events[self.expect_install_package],
events[self.expect_emerge_package]):
pkg_name = self.child.match.group(4).strip()
if res == events[self.expect_emerge_package]:
binary_pkg = bool(self.child.match.group(1))
current_pkg_num = int(self.child.match.group(2))
max_pkg_num = int(self.child.match.group(3))
self.notifyEmergingPackage(pkg_name, num=current_pkg_num,
max_num=max_pkg_num,
binary=binary_pkg)
else:
self.notifyInstallingPackage(pkg_name)
elif res == events[self.expect_error_on_install]:
if self.child.expect(
["The complete build log is located at '[^']+",
pexpect.EOF]) == 0:
self.error_log = self.child.after.rpartition("'")[-1]
raise EmergeError(_("Emerge %s is failed") % pkg_name)
else:
if self.child.isalive():
self.child.wait()
return self.child.exitstatus == 0
def execute(self):
# TODO: убрать лог в /tmp
if self.child is None:
self.child = pexpect.spawn(self.cmd, self.params + self.packages,
logfile=open('/tmp/emerge.log', 'w'),
env=self.env, cwd=self.cwd, timeout=None)
return self.child
def close(self):
if self.child is not None:
self.child.close()
self.child = None
def get_error_log(self):
"""
Получить путь до журнала сборки
"""
return self.error_log
def process_need_root(self):
raise EmergeError(
_("This action requires superuser access"),
errno=EmergeError.NEED_ROOT)
def __enter__(self):
return self
def __exit__(self, *exc_info):
self.close()
def get_update_list(self):
"""
Проверить есть ли в обновлениях пакет совпадающий с pkg_pattern
полученный пакет можно проверить указанной check функцией
"""
if self.update_packages is None:
list_block = XmlConverter().transform(self.get_update_block())
self.update_packages = self._parse_update_list(list_block)
return PackageList(self.update_packages)
def _parse_update_list(self, list_block):
return filter(lambda x: x['PN'] is not None,
map(EmergeUpdateInfo,
list_block.split('\n')))
class RevdepRebuild(Emerge):
def __init__(self):
pass
def execute(self):
pass
class EmergeLogTask(object):
def has_marker(self, line):
"""
Определить есть ли в строке маркер задачи
@ -1010,6 +814,7 @@ class EmergeLogTask(object):
"""
return ""
class EmergeLogNamedTask(EmergeLogTask):
date_format = "%b %d, %Y %T"
@ -1036,19 +841,23 @@ class EmergeLogNamedTask(EmergeLogTask):
"""
return "*** Finished %s" % self.taskname
class EmergeLog:
"""
EmergeLog(after).get_update(package_pattern)
"""
emerge_log = "/var/log/emerge.log"
re_complete_emerge = re.compile(r":::\scompleted emerge \(.*?\) (\S+)",
re_complete_emerge = re.compile(r":::\scompleted (emerge) \(.*?\) (\S+)",
re.M)
re_complete_unmerge = re.compile(r">>>\s(unmerge) success: (\S+)", re.M)
def __init__(self, emerge_task=EmergeLogTask()):
"""
@type emerge_task: EmergeLogTask
"""
self.emerge_task = emerge_task
self._list = None
self._remove_list = None
def _get_last_changes(self):
"""
@ -1060,16 +869,36 @@ class EmergeLog:
log_data.save()
return log_data.restore()
@property
def list(self):
if self._list is None:
self.get_packages()
return self._list
@property
def remove_list(self):
if self._remove_list is None:
self.get_packages()
return self._remove_list
def get_packages(self):
"""
Получить список пакетов
"""
return list(self._parse_log(self._get_last_changes()))
self._list, self._remove_list = \
zip(*self._parse_log(self._get_last_changes()))
self._list = filter(None,self._list)
self._remove_list = filter(None, self._remove_list)
def _parse_log(self, data):
for re_match in ifilter(None,
imap(self.re_complete_emerge.search, data)):
yield re_match.group(1)
searcher = lambda x:(self.re_complete_emerge.search(x) or
self.re_complete_unmerge.search(x))
for re_match in ifilter(None, imap(searcher, data)):
if re_match.group(1) == "emerge":
yield re_match.group(2), None
else:
yield None, re_match.group(2)
yield None, None
def _set_marker(self, text_marker):
with open(self.emerge_log, 'a') as f:
@ -1096,13 +925,18 @@ class PackageList(object):
"""
Список пакетов с возможностью среза и сравнением с версией
"""
def __init__(self, packages):
self._raw_list = packages
self.result = None
def _packages(self):
return ifilter(lambda x: x['PN'] is not None,
imap(lambda x: x if isinstance(x, Mapping) else EmergePackage(x),
self._raw_list))
if self.result is None:
self.result = filter(lambda x: x['PN'],
imap(lambda x: (x if isinstance(x, Mapping)
else EmergePackage(x)),
self._raw_list))
return self.result
def __getitem__(self, item):
re_item = re.compile(item)
@ -1113,7 +947,7 @@ class PackageList(object):
return iter(self._packages())
def __len__(self):
return len(self._raw_list)
return len(list(self._packages()))
def __lt__(self, other):
return any(pkg < other for pkg in self._packages())

323
update/update.py

@ -20,25 +20,42 @@ from os import path
from calculate.lib.utils.tools import AddonError
from calculate.lib.utils.colortext.palette import TextState
from calculate.lib.utils.colortext import get_color_print
import pexpect
from package_tools import Git, Layman, EmergeError, Emerge
from package_tools import Git, Layman,\
EmergeLogNamedTask, EmergeLog, GitError, \
PackageInformation
Colors = TextState.Colors
from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
PercentProgress, process, readFile)
from calculate.lib.utils.colortext import convert_console_to_xml
from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
PercentProgress, process)
from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
RegexpLocalization, _)
import emerge_parser
from emerge_parser import EmergeParser, EmergeCommand, EmergeError, EmergeCache
setLocalTranslate('cl_update3', sys.modules[__name__])
__ = getLazyLocalTranslate(_)
class UpdateError(AddonError):
"""Update Error"""
class Update:
"""Основной объект для выполнения действий связанных с обновлением системы
"""
def init(self):
commandLog = path.join(self.clVars.Get('core.cl_log_path'),
'lastcommand.log')
emerge_parser.CommandExecutor.logfile = commandLog
self.color_print = get_color_print()
self.emerge_cache = EmergeCache()
self.emerge_cache.check_list = (
self.emerge_cache.check_list +
map(emerge_parser.GitCheckvalue,
self.clVars.Get('update.cl_update_rep_path')))
def _syncRepository(self, name, url, rpath, revision, branch,
cb_progress=None):
@ -51,17 +68,17 @@ class Update:
if not git.checkExistsRep(rpath):
if revision == "last":
git.cloneRepository(url, rpath, branch,
cb_progress=cb_progress)
cb_progress=cb_progress)
else:
git.cloneRevRepository(url, rpath, branch, revision,
cb_progress=cb_progress)
cb_progress=cb_progress)
needMeta = True
else:
# если нужно обновиться до конкретной ревизии
if revision != "last":
if revision == git.getCurrentCommit(rpath):
if git.getBranch(rpath) == branch:
return True
return False
# получить изменения из удаленного репозитория
git.fetchRepository(rpath, cb_progress=cb_progress)
# если текущая ветка не соответствует нужной
@ -98,8 +115,9 @@ class Update:
self.addProgress()
if clean_on_error:
try:
self._syncRepository(repname, url, rpath, revision, branch,
cb_progress=self.setProgress)
if not self._syncRepository(repname, url, rpath, revision, branch,
cb_progress=self.setProgress):
return "skip"
return True
except GitError as e:
if e.addon:
@ -135,8 +153,9 @@ class Update:
else:
p = process(layman, "-s", repname, stderr=STDOUT)
if p.failed():
raise UpdateError(_("Failed to update repository {rname}"
).format(rname=repname), addon=p.read())
raise UpdateError(
_("Failed to update repository {rname}").format(rname=repname),
addon=p.read())
return True
def regenCache(self, repname):
@ -208,7 +227,7 @@ class Update:
def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
self.endTask()
_print = get_color_print()
_print = self.color_print
if max_num > 1:
one = _print.foreground(Colors.YELLOW).bold("{0}", num)
two = _print.foreground(Colors.YELLOW).bold("{0}", max_num)
@ -220,42 +239,278 @@ class Update:
else:
_print = _print.foreground(Colors.GREEN)
self.startTask("Emerging%s %s" % (part, _print(str(pkg))))
self.startTask(_("Emerging%s %s") % (part, _print(str(pkg))))
def _printInstallPackage(self, pkg, binary=False):
self.endTask()
_print = self.color_print
if binary:
_print = _print.foreground(Colors.PURPLE)
else:
_print = _print.foreground(Colors.GREEN)
self.startTask(_("Installing %s") %
_print(str(pkg)))
def _printInstallPackage(self, pkg):
def _printUninstallPackage(self, pkg, num=1, max_num=1):
self.endTask()
_print = get_color_print()
self.startTask(_("Installing %s")%
_print.foreground(Colors.YELLOW).bold(str(pkg)))
_print = self.color_print
if max_num > 1:
one = _print.foreground(Colors.YELLOW).bold("{0}", num)
two = _print.foreground(Colors.YELLOW).bold("{0}", max_num)
part = " (%s of %s)" % (one, two)
else:
part = ""
_print = _print.foreground(Colors.RED)
self.startTask(_("Unmerging%s %s") % (part, _print.bold(str(pkg))))
def emergelike(self, cmd, *params):
cmd_path = getProgPath(cmd)
if not cmd_path:
raise UpdateError(_("Failed to find %s command") % cmd)
with EmergeParser(
emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
emerge.emerging.add_observer(self._printEmergePackage)
emerge.installing.add_observer(self._printInstallPackage)
emerge.uninstalling.add_observer(self._printUninstallPackage)
try:
emerge.run()
except EmergeError:
self.printPre(self._emerge_translate(emerge.emerging_error.log))
raise
return True
def _display_pretty_package_list(self, pkglist, remove_list=False):
"""
Отобразить список пакетов в "удобночитаемом" виде
"""
_print = self.color_print
ebuild_color = TextState.Colors.GREEN
binary_color = TextState.Colors.PURPLE
remove_color = TextState.Colors.LIGHT_RED
for pkg in sorted([PackageInformation.add_info(x) for x in
pkglist],
key=lambda y: y['CATEGORY/PN']):
if remove_list:
pkgcolor = _print.foreground(remove_color)
else:
if pkg['binary']:
pkgcolor = _print.foreground(binary_color)
else:
pkgcolor = _print.foreground(ebuild_color)
fullname = _(pkg.info['DESCRIPTION']).capitalize()