Добавлен парсер Emerge и прочие вспомогательные файлы

setup.py

@@ -20,21 +20,21 @@
 __app__ = "calculate-update"
 __version__ = "3.1.8"
 
-import os
 from distutils.core import setup
 from calculate.install_data import install_data
 
-data_files = [('/usr/libexec/calculate',[('data/cl-git-wrapper',0755)])]
+data_files = [('/usr/libexec/calculate', [('data/cl-git-wrapper', 0755)])]
 
 setup(
-    name = __app__,
-    version = __version__,
-    description = "Update system utilities",
-    author = "Calculate Ltd.",
-    author_email = "support@calculate.ru",
-    url = "http://calculate-linux.org",
-    license = "http://www.apache.org/licenses/LICENSE-2.0",
-    package_dir = {'calculate.update': "update"},
-    data_files = data_files,
-    packages = ['calculate.update','calculate.update.utils','calculate.update.variables'],
+    name=__app__,
+    version=__version__,
+    description="Update system utilities",
+    author="Calculate Ltd.",
+    author_email="support@calculate.ru",
+    url="http://calculate-linux.org",
+    license="http://www.apache.org/licenses/LICENSE-2.0",
+    package_dir={'calculate.update': "update"},
+    data_files=data_files,
+    packages=['calculate.update', 'calculate.update.utils',
+              'calculate.update.variables'],
     cmdclass={'install_data': install_data})

update/emerge_parser.py

@@ -0,0 +1,603 @@
+#-*- coding: utf-8 -*-
+
+# Copyright 2014 Calculate Ltd. http://www.calculate-linux.org
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+from os import path
+
+import re
+import sys
+from calculate.lib.utils.colortext.palette import TextState
+from calculate.lib.utils.tools import ignore
+from package_tools import EmergePackage, PackageList, EmergeUpdateInfo, \
+    UnmergePackage, EmergeRemoveInfo, Git, GitError
+
+Colors = TextState.Colors
+import pexpect
+from calculate.lib.utils.files import getProgPath, readLinesFile, listDirectory, \
+    writeFile, readFile
+from calculate.lib.utils.colortext.output import XmlOutput
+from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter,
+                                                     XmlConverter)
+
+from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _
+
+setLocalTranslate('cl_update3', sys.modules[__name__])
+__ = getLazyLocalTranslate(_)
+
+
+class EmergeError(Exception):
+    """
+    Ошибка при сборке пакетов
+    """
+
+
+class EmergeNeedRootError(EmergeError):
+    pass
+
+
+class CommandExecutor(object):
+    """
+    Запуск программы для объекта Emerge
+    """
+    logfile = '/var/log/calculate/lastcommand.log'
+
+    def __init__(self, cmd, params, env=None, cwd=None, logfile=None):
+        self.cwd = cwd
+        self.env = env
+        self.cmd = cmd
+        self.params = params
+        self.child = None
+        if logfile:
+            self.logfile = logfile
+
+    def execute(self):
+        if self.child is None:
+            self.child = pexpect.spawn(self.cmd,
+                                       logfile=open(self.logfile, 'w'),
+                                       env=self.env, cwd=self.cwd, timeout=None)
+        return self.child
+
+    def close(self):
+        if self.child is not None:
+            self.child.close()
+            self.child = None
+
+    def success(self):
+        if self.child:
+            if self.child.isalive():
+                self.child.wait()
+            return self.child.exitstatus == 0
+        return False
+
+    def failed(self):
+        return not self.success()
+
+    def send(self, s):
+        if self.child:
+            self.child.send(s)
+
+
+class EmergeCommand(CommandExecutor):
+    """
+    Запуск emerge для последующего анализирования
+    """
+    # параметры по умолчанию
+    default_params = ["-av", "--color=y", "--nospinner"]
+    cmd = getProgPath("/usr/bin/emerge")
+
+    def __init__(self, packages, extra_params=None, env=None, cwd=None,
+                 logfile=None):
+        extra_params = extra_params or []
+        self.child = None
+        self.packages = packages
+        self.params = self.default_params + extra_params
+        default_env = {'CLEAN_DELAY': '0'}
+        default_env.update(os.environ)
+        self.env = env or default_env
+        self.cwd = cwd
+        if logfile:
+            self.logfile = logfile
+
+    def execute(self):
+        if self.child is None:
+            self.child = pexpect.spawn(self.cmd, self.params + self.packages,
+                                       logfile=open(self.logfile, 'w'),
+                                       env=self.env, cwd=self.cwd, timeout=None)
+        return self.child
+
+
+class EmergeInformationBlock(object):
+    _color_block = "(?:\033\[[^m]+?m)?"
+    _new_line = "\r*\n"
+    token = None
+    end_token = ["\n"]
+    re_block = None
+    action = None
+    re_match_type = type(re.match("", ""))
+    re_type = type(re.compile(""))
+
+    def __init__(self, parent):
+        """
+        :type parent: EmergeParser
+        """
+        self.result = None
+        self.text_converter = parent.text_converter
+        self.parent = parent
+        self.parent.add_element(self)
+        self.children = []
+
+    def add_element(self, element):
+        self.children.append(element)
+
+    def __str__(self):
+        if type(self.result) == self.re_match_type:
+            return self.result.group()
+        else:
+            return self.result or ""
+
+    def __nonzero__(self):
+        return bool(self.result)
+
+    def __len__(self):
+        if self.result is None:
+            return 0
+        else:
+            return len(self.result)
+
+    def __contains__(self, item):
+        if self.result is None:
+            return 0
+        else:
+            return item in self.result
+
+    def _get_text(self, result):
+        """
+        Получить результат из регулярки и преобразовать его через self.converter
+        """
+        if result:
+            return self.text_converter.transform(result.rstrip())
+        return ""
+
+    def get_block(self, child):
+        try:
+            token = child.match
+            if type(self.end_token) == self.re_type:
+                child.expect(self.end_token)
+                match = child.match.group()
+            else:
+                child.expect_exact(self.end_token)
+                match = child.match
+            self.get_data(self.re_block.search(
+                token + child.before + match))
+        except pexpect.EOF:
+            child.buffer = "".join(
+                [x for x in (child.before, child.after, child.buffer)
+                 if type(x) == str])
+
+    def get_data(self, match):
+        self.result = self._get_text(match.group(1))
+
+
+class InstallPackagesBlock(EmergeInformationBlock):
+    """
+    Блок emerge содержащий список пакетов для установки
+    """
+    list = PackageList([])
+    remove_list = PackageList([])
+    _new_line = EmergeInformationBlock._new_line
+    token = "\n["
+    end_token = ["\r\n\r", "\n\n"]
+
+    re_block = re.compile(r"((?:^\[.*?{nl})+)".format(nl=_new_line),
+                          re.MULTILINE)
+
+    def get_data(self, match):
+        super(InstallPackagesBlock, self).get_data(match)
+        list_block = XmlConverter().transform(self.result).split('\n')
+        self.list = PackageList(map(EmergeUpdateInfo, list_block))
+        self.remove_list = PackageList(map(EmergeRemoveInfo, list_block))
+
+
+class UninstallPackagesBlock(EmergeInformationBlock):
+    """
+    Блок emerge содержащий список удаляемых пакетов
+    """
+    list = PackageList([])
+    _new_line = EmergeInformationBlock._new_line
+    _color_block = EmergeInformationBlock._color_block
+    token = ["These are the packages that would be unmerged",
+             "Calculating removal order"]
+    end_token = re.compile("All selected packages:.*\n")
+    re_block = re.compile(r"All selected packages: (.*?){nl}".
+                          format(nl=_new_line, c=_color_block), re.DOTALL)
+
+    def get_data(self, match):
+        super(UninstallPackagesBlock, self).get_data(match)
+        list_block = XmlConverter().transform(self.result).split()
+        self.list = PackageList(map(EmergePackage, list_block))
+
+
+class FinishEmergeGroup(EmergeInformationBlock):
+    """
+    Блок завершения команды
+    """
+    token = pexpect.EOF
+    # регуляреное выражение, определяющее содержит ли блок
+    # сообщения об ошибках
+    re_failed = re.compile(
+        r"Fetch instructions for \S+:|"
+        r"The following.*are necessary to proceed|"
+        r"!!! Multiple package .* slot have been pulled|"
+        r"no ebuilds to satisfy|"
+        r"Dependencies could not be completely resolved due to",
+        re.MULTILINE)
+
+    def get_block(self, child):
+        if child.isalive():
+            child.wait()
+        if child.exitstatus != 0 or self.re_failed.search(child.before):
+            self.children_get_block(child)
+        else:
+            self.result = True
+
+    def children_get_block(self, child):
+        for block in self.children:
+            block.get_block(child)
+
+    def children_action(self, child):
+        for block in (x for x in self.children if x.result and x.action):
+            if block.action(child) is False:
+                break
+
+    def action(self, child):
+        self.children_action(child)
+        return False
+
+
+class PrepareErrorBlock(EmergeInformationBlock):
+    """
+    Блок информации с ошибками при получении списка устанавливаемых пакетов
+    """
+    token = None
+
+    re_drop = re.compile("news items need reading|"
+                         "Use eselect news|"
+                         "Calculating dependencies|"
+                         "to read news items|"
+                         "Local copy of remote index is up-to-date|"
+                         "These are the packages that would be merged|"
+                         "Process finished with exit code")
+    re_multi_empty_line = re.compile("(?:<br/>){3,}", re.DOTALL)
+    re_strip_br = re.compile("^(?:<br/>)+|(?:<br/>)+$", re.DOTALL)
+
+    def remove_needless_data(self, data):
+        return "\n".join([x for x in data.split('\n')
+                          if not self.re_drop.search(x)])
+
+    def strip_br(self, data):
+        return self.re_strip_br.sub(
+            "",
+            self.re_multi_empty_line.sub("<br/><br/>", data))
+
+    def get_block(self, child):
+        self.result = self.strip_br(
+            self._get_text(self.remove_needless_data(child.before)))
+
+    def action(self, child):
+        raise EmergeError(_("Emerge failed"))
+
+
+class DownloadSizeBlock(EmergeInformationBlock):
+    """
+    Размер скачиваемых обновлений
+    """
+    token = "Size of downloads:"
+
+    re_block = re.compile(r"Size of downloads:\s(\S+\s\S+)")
+
+    def __str__(self):
+        if self.result:
+            return self.result
+        else:
+            return "0 kB"
+
+
+class QuestionBlock(EmergeInformationBlock):
+    """
+    Блок вопроса
+    """
+    default_answer = "no"
+    _color_block = EmergeInformationBlock._color_block
+    token = "Would you like"
+    end_token = ["]", "\n"]
+    re_block = re.compile(
+        "(Would you.*)\[{c}Yes{c}/{c}No{c}".format(c=_color_block))
+
+    def action(self, child):
+        if self.result:
+            child.send("%s\n" % self.default_answer)
+
+
+class NeedRootBlock(EmergeInformationBlock):
+    """
+    Пользователь не явеляется root
+    """
+    token = "This action requires superuser access"
+
+    def get_data(self, child):
+        self.result = True
+
+    def action(self, child):
+        raise EmergeNeedRootError(_("This action requires superuser access"))
+
+
+class NotifierInformationBlock(EmergeInformationBlock):
+    """
+    Информационный блок поддерживающий observing
+    """
+    def __init__(self, parent):
+        super(NotifierInformationBlock, self).__init__(parent)
+        self.observers = []
+
+    def get_data(self, match):
+        self.result = match
+
+    def add_observer(self, f):
+        self.observers.append(f)
+
+    def notify(self, observer, groups):
+        observer(groups)
+
+    def action(self, child):
+        if self.result and self.observers:
+            groups = self.result.groups()
+            for observer in self.observers:
+                self.notify(observer, groups)
+
+
+class EmergingPackage(NotifierInformationBlock):
+    """
+    Запуск устанавливаемого пакета
+
+    ObserverFunc: (package, num=number, max_num=number, binary=binary)
+    """
+    _color_block = EmergeInformationBlock._color_block
+    token = ">>> Emerging "
+    re_block = re.compile(
+        "Emerging (binary )?\({c}(\d+){c} "
+        "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
+
+    def notify(self, observer, groups):
+        observer(EmergePackage(groups[3]), num=groups[1], max_num=groups[2],
+                 binary=bool(groups[0]))
+
+
+class UnemergingPackage(NotifierInformationBlock):
+    """
+    Запуск устанавливаемого пакета
+
+    ObserverFunc: (package, num=number, max_num=number)
+    """
+    _color_block = EmergeInformationBlock._color_block
+    token = ">>> Unmerging"
+    re_block = re.compile(
+        r"Unmerging (?:\({c}(\d+){c} "
+        r"of {c}(\d+){c}\) )?(\S+)\.\.\.".format(c=_color_block))
+
+    def notify(self, observer, groups):
+        observer(EmergePackage(groups[2]), num=groups[0], max_num=groups[1])
+
+
+class InstallingPackage(NotifierInformationBlock):
+    """
+    Запуск устанавливаемого пакета
+
+    ObserverFunc: (package, binary=binary)
+    """
+    _color_block = EmergeInformationBlock._color_block
+    binary = None
+
+    token = ">>> Installing "
+    re_block = re.compile(
+        "Installing \({c}(\d+){c} "
+        "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
+
+    def notify(self, observer, groups):
+        binary = bool(self.binary and groups[2] in self.binary)
+        observer(EmergePackage(groups[2]), binary=binary)
+
+    def mark_binary(self, package):
+        if self.binary is None:
+            self.binary = []
+        self.binary.append(str(package))
+
+
+class EmergeingErrorBlock(EmergeInformationBlock):
+    """
+    Блок содержит информацию об ошибке во время сборки пакета
+    """
+    token = ["* ERROR: ", " * \033[39;49;00mERROR: "]
+    end_token = "Working directory:"
+    re_block = re.compile("ERROR: (\S*) failed \([^)]+\).*?"
+                          "The complete build log is located at '([^']+)",
+                          re.DOTALL)
+    package = ""
+
+    def get_data(self, match):
+        self.result = self._get_text(match.group(2).rstrip())
+        self.package = match.group(1)
+
+    @property
+    def log(self):
+        return self.text_converter.transform(readFile(self.result))
+
+    def action(self, child):
+        raise EmergeError(_("Emerge %s is failed") % self.package)
+
+
+class EmergeParser(object):
+    """
+    Парсер вывода emerge
+    """
+
+    def __init__(self, command, run=False):
+        self.text_converter = ConsoleCodes256Converter(XmlOutput())
+        self.command = command
+        self.elements = {}
+
+        self.install_packages = InstallPackagesBlock(self)
+        self.uninstall_packages = UninstallPackagesBlock(self)
+        self.question = QuestionBlock(self)
+        self.finish_block = FinishEmergeGroup(self)
+        self.need_root = NeedRootBlock(self)
+        self.prepare_error = PrepareErrorBlock(self.finish_block)
+        self.download_size = DownloadSizeBlock(self)
+        self.emerging_error = EmergeingErrorBlock(self)
+
+        self.installing = InstallingPackage(self)
+        self.uninstalling = UnemergingPackage(self)
+        self.emerging = EmergingPackage(self)
+
+        self.emerging.add_observer(self.mark_binary)
+        if run:
+            self.run()
+
+    def mark_binary(self, package, binary=False, **kw):
+        if binary:
+            self.installing.mark_binary(package)
+
+    def add_element(self, element):
+        if element.token:
+            if type(element.token) == list:
+                for token in element.token:
+                    self.elements[token] = element
+            else:
+                self.elements[element.token] = element
+
+    def run(self):
+        """
+        Запустить команду
+        """
+        child = self.command.execute()
+
+        while True:
+            index = child.expect_exact(self.elements.keys())
+            element = self.elements.values()[index]
+            element.get_block(child)
+            if element.action:
+                if element.action(child) is False:
+                    break
+
+    def close(self):
+        self.command.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        self.close()
+
+
+class MtimeCheckvalue(object):
+    def __init__(self, *fname):
+        self.fname = fname
+
+    def value_func(self, fn):
+        return str(int(os.stat(fn).st_mtime))
+
+    def get_check_values(self, file_list):
+        for fn in file_list:
+            if path.exists(fn) and not path.isdir(fn):
+                yield fn, self.value_func(fn)
+            else:
+                for k, v in self.get_check_values(
+                        listDirectory(fn, fullPath=True)):
+                    yield k, v
+
+    def checkvalues(self):
+        return self.get_check_values(self.fname)
+
+
+class GitCheckvalue(object):
+    def __init__(self, rpath):
+        self.rpath = rpath
+        self.git = Git()
+
+    def checkvalues(self):
+        with ignore(GitError):
+            if self.git.is_git(self.rpath):
+                yield self.rpath, Git().getCurrentCommit(self.rpath)
+
+
+class EmergeCache(object):
+    """
+    Кэш пакетов
+    """
+    cache_file = '/var/lib/calculate/calculate-update/world.cache'
+    # список файлов проверяемый по mtime на изменения
+    check_list = [MtimeCheckvalue('/etc/make.conf',
+                  '/etc/portage',
+                  '/etc/make.profile',
+                  '/var/cache/edb/binhost')]
+
+    def __init__(self):
+        self.files_control_values = {}
+        self.pkg_list = PackageList([])
+
+    def set_cache(self, package_list):
+        """
+        Установить в кэш список пакетов
+        """
+        with writeFile(self.cache_file) as f:
+            for fn, val in self.get_control_values().items():
+                f.write("{fn}={val}\n".format(fn=fn, val=val))
+            f.write('\n')
+            for pkg in package_list:
+                f.write("%s\n"% str(pkg))
+
+    def drop_cache(self):
+        if path.exists(self.cache_file):
+            with ignore(OSError):
+                os.unlink(self.cache_file)
+
+    def get_cached_package_list(self):
+        self.read_cache()
+        if self.check_actuality():
+            return self.pkg_list
+        return None
+
+    def check_actuality(self):
+        """
+        Кэш считается актуальным если ни один из файлов не менялся
+        """
+        return self.get_control_values() == self.files_control_values
+
+    def read_cache(self):
+        self.files_control_values = {}
+        cache_file_lines = readLinesFile(self.cache_file)
+        for line in cache_file_lines:
+            if not "=" in line:
+                break
+            k, v = line.split('=')
+            self.files_control_values[k] = v.strip()
+        self.pkg_list = PackageList(cache_file_lines)
+
+    def get_control_values(self):
+        def generate():
+            for obj in self.check_list:
+                for checkvalue in obj.checkvalues():
+                    yield checkvalue
+        return dict(generate())

update/package_tools.py

@@ -13,7 +13,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from collections import Mapping
+from collections import Mapping, defaultdict
 
 import re
 import sys
@@ -28,12 +28,10 @@ from os import path
 from calculate.lib.utils.files import (getProgPath, STDOUT,
                                        PercentProgress, process, readFile,
                                        readLinesFile)
-from calculate.lib.utils.colortext.output import XmlOutput
-from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter,
-                                                     XmlConverter)
 from calculate.lib.utils.common import cmpVersion
 from contextlib import closing
 
+
 import xml.etree.ElementTree as ET
 
 from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
@@ -127,7 +125,6 @@ class Layman:
         self._add_to_makeconf(rpath)
         return True
 
-
 class Git:
     """
     Объект для управление git репозиторием
@@ -487,7 +484,7 @@ class EmergePackage(Mapping):
     pv = r"(?:-(\d[^-]*?))?"
     pr = r"(?:-(r\d+))?"
     tbz = r"(?:.(tbz2))?"
-    slot = r'(?::(\d+(?:\.\d+)*(?:/\d+(?:\.\d+))?))?'
+    slot = r'(?::(\w+(?:\.\w+)*(?:/\w+(?:\.\w+)*)?))?'
     repo = r'(?:::(\w+))?'
 
     reParse = re.compile(
@@ -515,7 +512,10 @@ class EmergePackage(Mapping):
                  'REPO': x[REPO] or self.default_repo,
                  'CATEGORY/PN': "%s/%s" % (x[CATEGORY], x[PN]),
                  'PR': x[PR] or 'r0'}
-            d['PVR'] = "%s-%s" % (d['PV'], d['PR'])
+            if x[PR]:
+                d['PVR'] = "%s-%s" % (d['PV'], d['PR'])
+            else:
+                d['PVR'] = d['PV']
             if d['PF'].endswith('-r0'):
                 d['PF'] = d['PF'][:-3]
             return d.copy()
@@ -538,27 +538,31 @@ class EmergePackage(Mapping):
         if "CATEGORY/PN" in version and "PVR" in version:
             if self['CATEGORY/PN'] < version['CATEGORY/PN']:
                 return True
-            version = version['PVR']
-        return cmpVersion(self['PVR'], version) == -1
+            elif self['CATEGORY/PN'] > version['CATEGORY/PN']:
+                return False
+            version = "%s-%s" % (version['PV'], version['PR'])
+        currentVersion = "%s-%s" % (self['PV'], self['PR'])
+        return cmpVersion(currentVersion, version) == -1
 
     def __eq__(self, version):
         if "CATEGORY" in version and "PF" in version:
             return ("%s/%s" % (self['CATEGORY'], self['PF']) ==
                     "%s/%s" % (version['CATEGORY'], version['PF']))
         else:
-            return cmpVersion(self['PVR'], version) == 0
+            currentVersion = "%s-%s" % (self['PV'], self['PR'])
+            return cmpVersion(currentVersion, version) == 0
 
     def __init__(self, package):
         if isinstance(package, EmergePackage):
             self.__result = package.__result
-            self.__package = package.__package
+            self._package = package._package
         else:
-            self.__package = package
+            self._package = package
             self.__result = None
 
     def __getitem__(self, item):
         if not self.__result:
-            self.__result = self._parsePackageString(self.__package)
+            self.__result = self._parsePackageString(self._package)
         return self.__result[item]
 
     def __repr__(self):
@@ -568,6 +572,85 @@ class EmergePackage(Mapping):
         return "%s/%s" % (self['CATEGORY'], self['PF'])
 
 
+class PackageInformation:
+    """
+    Объект позволяет получать информацию о пакете из eix
+    """
+    eix_cmd = getProgPath("/usr/bin/eix")
+
+    query_packages = []
+    information_cache = defaultdict(dict)
+
+    fields = ["DESCRIPTION"]
+
+    def __init__(self, pkg):
+        self._pkg = pkg
+        if not pkg in self.query_packages:
+            self.query_packages.append(pkg)
+
+    def __getitem__(self, item):
+        if not self._pkg['CATEGORY/PN'] in self.information_cache:
+            self.query_information()
+        try:
+            return self.information_cache[self._pkg['CATEGORY/PN']][item]
+        except KeyError:
+            return ""
+
+    def query_information(self):
+        pkg_list = "|".join(
+            [x['CATEGORY/PN'].replace("+", r"\+") for x in self.query_packages])
+        output = pexpect.spawn(self.eix_cmd, ["--xml", pkg_list]).read()
+        xml = ET.fromstring(output)
+        for pkg in self.query_packages:
+            cat_pn = pkg['CATEGORY/PN']
+            if not cat_pn in self.information_cache:
+                descr_node = xml.find(
+                    'category[@name="%s"]/package[@name="%s"]/description'
+                    % (pkg['CATEGORY'], pkg['PN']))
+                if descr_node is not None:
+                    self.information_cache[cat_pn]['DESCRIPTION'] = \
+                        descr_node.text
+        while self.query_packages:
+            self.query_packages.pop()
+
+    @classmethod
+    def add_info(cls, pkg):
+        pkg.info = cls(pkg)
+        return pkg
+
+class UnmergePackage(EmergePackage):
+    """
+    Информация об обновлении одного пакета
+    """
+    re_pkg_info = re.compile("^\s(\S+)\n\s+selected:\s(\S+)",re.MULTILINE)
+
+    def __init__(self, package):
+        super(UnmergePackage, self).__init__(package)
+        if not isinstance(package, EmergePackage):
+            self._package = self.convert_package_info(package)
+
+    def convert_package_info(self, package):
+        match = self.re_pkg_info.search(package)
+        if match:
+            return "%s-%s" % match.groups()
+        return ""
+
+
+def recalculate_update_info(cls):
+    """
+    Добавить
+    """
+    cls.update_info = re.compile(
+        r"^({install_info})\s+({atom_info})\s*(?:{prev_version})?"
+        r"\s*({use_info})?.*?({pkg_size})?$".format(
+            install_info=cls.install_info,
+            atom_info=cls.atom_info,
+            prev_version=cls.prev_version,
+            use_info=cls.use_info,
+            pkg_size=cls.pkg_size), re.MULTILINE)
+    return cls
+
+@recalculate_update_info
 @total_ordering
 class EmergeUpdateInfo(Mapping):
     """
@@ -578,27 +661,22 @@ class EmergeUpdateInfo(Mapping):
     atom_info = r"\S+"
     use_info = 'USE="[^"]+"'
     prev_version = "\[([^\]]+)\]"
-    update_info = re.compile(
-        r"^({install_info})\s+({atom_info})\s+(?:{prev_version})?"
-        r"\s*({use_info})?".format(install_info=install_info,
-                                   atom_info=atom_info,
-                                   prev_version=prev_version,
-                                   use_info=use_info))
+    pkg_size = r"\d+ \w+"
 
-    attrs = ['binary', 'REPLACING_VERSIONS']
+    attrs = ['binary', 'REPLACING_VERSIONS', 'SIZE']
 
     def __init__(self, data):
         self._data = data
         self._package = None
         self._info = {}
 
-
     def _parseData(self):
         r = self.update_info.search(self._data)
         if r:
             self._info['binary'] = r.group(2) == 'binary'
             self._package = EmergePackage(r.group(3))
             self._info['REPLACING_VERSIONS'] = r.group(4) or ""
+            self._info['SIZE'] = r.group(6) or ""
 
     def __iter__(self):
         return chain(EmergePackage.attrs, self.attrs)
@@ -639,16 +717,12 @@ class EmergeUpdateInfo(Mapping):
     def __str__(self):
         return "%s/%s" % (self['CATEGORY'], self['PF'])
 
-
-class EmergeError(Exception):
+@recalculate_update_info
+class EmergeRemoveInfo(EmergeUpdateInfo):
     """
-    Ошибка при сборке пакетов
+    Информация об удалении одного пакета (в списке обновляемых пакетов)
     """
-    (NEED_ROOT, NO_EBUILD, CONFLICT, CUSTOM) = range(0, 4)
-
-    def __init__(self, msg, errno=CUSTOM):
-        Exception.__init__(self, msg)
-        self.errno = errno
+    install_info = "\[(uninstall)[^\]]+\]"
 
 
 class Eix:
@@ -721,277 +795,7 @@ class Eix:
             return iter(())
 
 
-class Emerge:
-    """
-    Объект для выполнения сборки пакетов
-    """
-    # параметры по умолчанию
-    default_params = ["-av", "--color=y", "--nospinner"]
-    cmd = getProgPath("/usr/bin/emerge")
-    # шаблон выборки списка пакетов
-    install_pkgs_pattern = \
-        "(\\[[^\\]]+\\]\\s*(\\S+).*(?:{nl}|$))".format(nl="\r*\n")
-    re_install_pkgs = re.compile("^%s+" % install_pkgs_pattern,
-                                 re.MULTILINE)
-    # конфликтный пакет
-    _conflict_pkg = r"[\w-]+/[\w-]+:\d+"
-    # конкретная версия конфликтного пакета
-    _conflict_candidate = r"  \(.*"
-    # пакеты в которым необходим конфликтный пакет
-    _conflict_req = r"    \S.*"
-    # шаблон конфликтного блока
-    conflict_block = \
-        "({nl}{nl}({0}{nl}({1}{nl})+{nl})+)".format(_conflict_candidate,
-                                                    _conflict_req,
-                                                    nl="\r*\n")
-    re_conflict_block = re.compile(
-        "^{0}{conflict_block}([\w\W]+{conflict_block})*".format(
-            _conflict_pkg, conflict_block=conflict_block),
-        re.MULTILINE)
-    # шаблон устновки консольного цвета
-    _color_block = "(?:\033\[[^m]+?m)?"
-    re_custom_error = re.compile(
-        r"^Calculating dependencies.*{nl}".format(nl="\r*\n") +
-        "%s*" % install_pkgs_pattern +
-        r"(?P<result>[\w\W]+)", re.MULTILINE)
-
-    # regexp для удаления неиспользуемой информации
-    re_cut_blocks = \
-        re.compile(r"(It may be possible to solve.*?"
-                   "solve this conflict automatically\.|"
-                   "For more information, see.*?Gentoo Handbook\.)\s*", re.S)
-
-    expect_need_root = "This action requires superuser access"
-    expect_conflict = "Multiple package[ \w]+pulled"
-    expect_to_merge = "Would you like to merge"
-    expect_nothing_merge = ("Nothing to merge|No outdated packages|"
-                            "\s0 packages")
-    expect_no_ebuild = "no ebuilds to satisfy \""
-    expect_custom_question = "{c}Yes{c}/{c}No{c}".format(c=_color_block)
-    expect_emerge_package = ("Emerging (binary )?\({c}(\d+){c} "
-                             "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(
-        c=_color_block))
-    expect_install_package = ("Installing (binary )?\({c}(\d+){c} "
-                              "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(
-        c=_color_block))
-    expect_error_on_install = "ERROR: \S* failed \([^)]+\)"
-
-    def __init__(self, packages, extra_params=[], env=None, cwd=None,
-                 command_runner=None,
-                 converter=ConsoleCodes256Converter(XmlOutput())):
-        self.text_converter = converter
-        self.packages = packages
-        self.params = self.default_params + extra_params
-        self.child = None
-        self.update_block = None
-        self.conflict_block = None
-        self.update_packages = None
-        self.custom_error = ""
-        self.env = env
-        self.cwd = cwd
-        # событие начало сборки
-        self.event_emerging = []
-        # событие начало установки
-        self.event_installing = []
-        self.error_log = None
-
-    def handle_emerging(self, f):
-        """
-        Добавить обработку начала сборки пакета
-        """
-        if not f in self.event_emerging:
-            self.event_emerging.append(f)
-
-    def handle_installing(self, f):
-        """
-        Добавить обработку начала установки пакета
-        """
-        if not f in self.event_installing:
-            self.event_installing.append(f)
-
-    def _get_text(self, re_result, already_text=False):
-        """
-        Получить результат из регулярки и преобразовать его через self.converter
-        """
-        if re_result:
-            if not already_text:
-                re_result = re_result.group()
-            result = self.re_cut_blocks.sub("", re_result)
-            return self.text_converter.transform(result.strip())
-        return ""
-
-    def _get_custom_error_output(self, buf):
-        """
-        Получить вывод об прочих ошибках в ходе вычисления зависимостей
-        """
-        r = self.re_custom_error.search(buf)
-        if r:
-            return self._get_text(r.groupdict()["result"], already_text=True)
-        return ""
-
-    def _get_package_list_output(self, buf):
-        """
-        Получить из вывода emerge часть текста со списком пакетов
-        """
-        return self._get_text(self.re_install_pkgs.search(buf))
-
-    def _get_conflict_list_output(self, buf):
-        """
-        Разобрать вывод о конфликтах среди пакетов
-        """
-        return self._get_text(self.re_conflict_block.search(buf))
-
-    def get_update_block(self):
-        """
-        Получить список устанавливаемых пакетов
-        """
-        if self.update_block is None:
-            self.execute()
-            events = {v: k for k, v in enumerate(sorted([
-                self.expect_need_root,
-                self.expect_conflict,
-                self.expect_to_merge,
-                self.expect_nothing_merge,
-                self.expect_no_ebuild,
-                self.expect_custom_question,
-                pexpect.EOF]))}
-            res = self.child.expect(sorted(events.keys()))
-            if res == events[self.expect_need_root]:
-                self.process_need_root()
-            elif res == events[self.expect_to_merge]:
-                self.update_block = (
-                    self._get_package_list_output(self.child.before))
-            elif res == events[self.expect_nothing_merge]:
-                self.update_block = ""
-            elif res == events[self.expect_no_ebuild]:
-                raise EmergeError(_("No ebuilds to satisfy %s") %
-                                  self.child.buffer.split('"', 1)[0],
-                                  errno=EmergeError.NO_EBUILD)
-            elif res == events[self.expect_conflict]:
-                self.update_block = \
-                    self._get_package_list_output(self.child.before)
-                self.conflict_block = \
-                    self._get_conflict_list_output(self.child.read())
-                raise EmergeError(_("Multiple package instances within "
-                                    "a single package slot"),
-                                  errno=EmergeError.CONFLICT)
-            elif res == events[self.expect_custom_question]:
-                self.custom_error = self._get_custom_error_output(
-                    self.child.before)
-                self.child.sendline("no")
-                raise EmergeError(_("Unexpected question"))
-            elif res == events[pexpect.EOF]:
-                if re.search("Total:\s+\d+ package", self.child.before):
-                    self.update_block = (
-                        self._get_package_list_output(self.child.before))
-                else:
-                    self.custom_error = self._get_custom_error_output(
-                        self.child.before)
-                    raise EmergeError(_("Failed to emerge"))
-        return self.update_block
-
-    def notifyEmergingPackage(self, package, num=1, max_num=2, binary=False):
-        [f(EmergePackage(package), num=num, max_num=max_num, binary=binary)
-         for f in self.event_emerging]
-
-    def notifyInstallingPackage(self, package):
-        [f(EmergePackage(package)) for f in self.event_installing]
-
-    def install(self):
-        """
-        Install packages
-        """
-        self.execute()
-        self.child.sendline("yes")
-        pkg_name = ""
-        events = {v: k for k, v in enumerate(sorted([
-            self.expect_need_root,
-            self.expect_emerge_package,
-            self.expect_install_package,
-            self.expect_error_on_install,
-            pexpect.EOF]))}
-        while True:
-            res = self.child.expect(sorted(events.keys()))
-            if res == events[self.expect_need_root]:
-                self.process_need_root()
-            elif res in (events[self.expect_install_package],
-                         events[self.expect_emerge_package]):
-                pkg_name = self.child.match.group(4).strip()
-                if res == events[self.expect_emerge_package]:
-                    binary_pkg = bool(self.child.match.group(1))
-                    current_pkg_num = int(self.child.match.group(2))
-                    max_pkg_num = int(self.child.match.group(3))
-                    self.notifyEmergingPackage(pkg_name, num=current_pkg_num,
-                                               max_num=max_pkg_num,
-                                               binary=binary_pkg)
-                else:
-                    self.notifyInstallingPackage(pkg_name)
-            elif res == events[self.expect_error_on_install]:
-                if self.child.expect(
-                        ["The complete build log is located at '[^']+",
-                         pexpect.EOF]) == 0:
-                    self.error_log = self.child.after.rpartition("'")[-1]
-                raise EmergeError(_("Emerge %s is failed") % pkg_name)
-            else:
-                if self.child.isalive():
-                    self.child.wait()
-                return self.child.exitstatus == 0
-
-    def execute(self):
-        # TODO: убрать лог в /tmp
-        if self.child is None:
-            self.child = pexpect.spawn(self.cmd, self.params + self.packages,
-                                       logfile=open('/tmp/emerge.log', 'w'),
-                                       env=self.env, cwd=self.cwd, timeout=None)
-        return self.child
-
-    def close(self):
-        if self.child is not None:
-            self.child.close()
-            self.child = None
-
-    def get_error_log(self):
-        """
-        Получить путь до журнала сборки
-        """
-        return self.error_log
-
-    def process_need_root(self):
-        raise EmergeError(
-            _("This action requires superuser access"),
-            errno=EmergeError.NEED_ROOT)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *exc_info):
-        self.close()
-
-    def get_update_list(self):
-        """
-        Проверить есть ли в обновлениях пакет совпадающий с pkg_pattern
-        полученный пакет можно проверить указанной check функцией
-        """
-        if self.update_packages is None:
-            list_block = XmlConverter().transform(self.get_update_block())
-            self.update_packages = self._parse_update_list(list_block)
-        return PackageList(self.update_packages)
-
-    def _parse_update_list(self, list_block):
-        return filter(lambda x: x['PN'] is not None,
-                      map(EmergeUpdateInfo,
-                          list_block.split('\n')))
-
-
-class RevdepRebuild(Emerge):
-    def __init__(self):
-        pass
-
-    def execute(self):
-        pass
-
 class EmergeLogTask(object):
-
     def has_marker(self, line):
         """
         Определить есть ли в строке маркер задачи
@@ -1010,6 +814,7 @@ class EmergeLogTask(object):
         """
         return ""
 
+
 class EmergeLogNamedTask(EmergeLogTask):
     date_format = "%b %d, %Y %T"
 
@@ -1036,19 +841,23 @@ class EmergeLogNamedTask(EmergeLogTask):
         """
         return "*** Finished %s" % self.taskname
 
+
 class EmergeLog:
     """
     EmergeLog(after).get_update(package_pattern)
     """
     emerge_log = "/var/log/emerge.log"
-    re_complete_emerge = re.compile(r":::\scompleted emerge \(.*?\) (\S+)",
+    re_complete_emerge = re.compile(r":::\scompleted (emerge) \(.*?\) (\S+)",
                                     re.M)
+    re_complete_unmerge = re.compile(r">>>\s(unmerge) success: (\S+)", re.M)
 
     def __init__(self, emerge_task=EmergeLogTask()):
         """
         @type emerge_task: EmergeLogTask
         """
         self.emerge_task = emerge_task
+        self._list = None
+        self._remove_list = None
 
     def _get_last_changes(self):
         """
@@ -1060,16 +869,36 @@ class EmergeLog:
                 log_data.save()
         return log_data.restore()
 
+    @property
+    def list(self):
+        if self._list is None:
+            self.get_packages()
+        return self._list
+
+    @property
+    def remove_list(self):
+        if self._remove_list is None:
+            self.get_packages()
+        return self._remove_list
+
     def get_packages(self):
         """
         Получить список пакетов
         """
-        return list(self._parse_log(self._get_last_changes()))
+        self._list, self._remove_list = \
+            zip(*self._parse_log(self._get_last_changes()))
+        self._list = filter(None,self._list)
+        self._remove_list = filter(None, self._remove_list)
 
     def _parse_log(self, data):
-        for re_match in ifilter(None,
-                                imap(self.re_complete_emerge.search, data)):
-            yield re_match.group(1)
+        searcher = lambda x:(self.re_complete_emerge.search(x) or
+                            self.re_complete_unmerge.search(x))
+        for re_match in ifilter(None, imap(searcher, data)):
+            if re_match.group(1) == "emerge":
+                yield re_match.group(2), None
+            else:
+                yield None, re_match.group(2)
+        yield None, None
 
     def _set_marker(self, text_marker):
         with open(self.emerge_log, 'a') as f:
@@ -1096,13 +925,18 @@ class PackageList(object):
     """
     Список пакетов с возможностью среза и сравнением с версией
     """
+
     def __init__(self, packages):
         self._raw_list = packages
+        self.result = None
 
     def _packages(self):
-        return ifilter(lambda x: x['PN'] is not None,
-               imap(lambda x: x if isinstance(x, Mapping) else EmergePackage(x),
-               self._raw_list))
+        if self.result is None:
+            self.result = filter(lambda x: x['PN'],
+                                  imap(lambda x: (x if isinstance(x, Mapping)
+                                                  else EmergePackage(x)),
+                                       self._raw_list))
+        return self.result
 
     def __getitem__(self, item):
         re_item = re.compile(item)
@@ -1113,7 +947,7 @@ class PackageList(object):
         return iter(self._packages())
 
     def __len__(self):
-        return len(self._raw_list)
+        return len(list(self._packages()))
 
     def __lt__(self, other):
         return any(pkg < other for pkg in self._packages())

update/update.py

@@ -20,25 +20,42 @@ from os import path
 from calculate.lib.utils.tools import AddonError
 from calculate.lib.utils.colortext.palette import TextState
 from calculate.lib.utils.colortext import get_color_print
+import pexpect
 
-from package_tools import Git, Layman, EmergeError, Emerge
+from package_tools import Git, Layman,\
+    EmergeLogNamedTask, EmergeLog, GitError, \
+    PackageInformation
 
 Colors = TextState.Colors
 from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
-                                       PercentProgress, process, readFile)
-from calculate.lib.utils.colortext import convert_console_to_xml
-from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
+                                       PercentProgress, process)
+from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
+                                   RegexpLocalization, _)
+import emerge_parser
+from emerge_parser import EmergeParser, EmergeCommand, EmergeError, EmergeCache
 
 setLocalTranslate('cl_update3', sys.modules[__name__])
 __ = getLazyLocalTranslate(_)
 
+
 class UpdateError(AddonError):
     """Update Error"""
 
+
 class Update:
     """Основной объект для выполнения действий связанных с обновлением системы
 
     """
+    def init(self):
+        commandLog = path.join(self.clVars.Get('core.cl_log_path'),
+                               'lastcommand.log')
+        emerge_parser.CommandExecutor.logfile = commandLog
+        self.color_print = get_color_print()
+        self.emerge_cache = EmergeCache()
+        self.emerge_cache.check_list = (
+            self.emerge_cache.check_list +
+            map(emerge_parser.GitCheckvalue,
+                self.clVars.Get('update.cl_update_rep_path')))
 
     def _syncRepository(self, name, url, rpath, revision, branch,
                         cb_progress=None):
@@ -51,17 +68,17 @@ class Update:
         if not git.checkExistsRep(rpath):
             if revision == "last":
                 git.cloneRepository(url, rpath, branch,
-                                     cb_progress=cb_progress)
+                                    cb_progress=cb_progress)
             else:
                 git.cloneRevRepository(url, rpath, branch, revision,
-                                        cb_progress=cb_progress)
+                                       cb_progress=cb_progress)
             needMeta = True
         else:
             # если нужно обновиться до конкретной ревизии
             if revision != "last":
                 if revision == git.getCurrentCommit(rpath):
                     if git.getBranch(rpath) == branch:
-                        return True
+                        return False
             # получить изменения из удаленного репозитория
             git.fetchRepository(rpath, cb_progress=cb_progress)
             # если текущая ветка не соответствует нужной
@@ -98,8 +115,9 @@ class Update:
         self.addProgress()
         if clean_on_error:
             try:
-                self._syncRepository(repname, url, rpath, revision, branch,
-                                     cb_progress=self.setProgress)
+                if not self._syncRepository(repname, url, rpath, revision, branch,
+                                     cb_progress=self.setProgress):
+                    return "skip"
                 return True
             except GitError as e:
                 if e.addon:
@@ -135,8 +153,9 @@ class Update:
         else:
             p = process(layman, "-s", repname, stderr=STDOUT)
         if p.failed():
-            raise UpdateError(_("Failed to update repository {rname}"
-            ).format(rname=repname), addon=p.read())
+            raise UpdateError(
+                _("Failed to update repository {rname}").format(rname=repname),
+                addon=p.read())
         return True
 
     def regenCache(self, repname):
@@ -208,7 +227,7 @@ class Update:
 
     def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
         self.endTask()
-        _print = get_color_print()
+        _print = self.color_print
         if max_num > 1:
             one = _print.foreground(Colors.YELLOW).bold("{0}", num)
             two = _print.foreground(Colors.YELLOW).bold("{0}", max_num)
@@ -220,42 +239,278 @@ class Update:
         else:
             _print = _print.foreground(Colors.GREEN)
 
-        self.startTask("Emerging%s %s" % (part, _print(str(pkg))))
+        self.startTask(_("Emerging%s %s") % (part, _print(str(pkg))))
+
+    def _printInstallPackage(self, pkg, binary=False):
+        self.endTask()
+        _print = self.color_print
+        if binary:
+            _print = _print.foreground(Colors.PURPLE)
+        else:
+            _print = _print.foreground(Colors.GREEN)
+        self.startTask(_("Installing %s") %
+                       _print(str(pkg)))
 
-    def _printInstallPackage(self, pkg):
+    def _printUninstallPackage(self, pkg, num=1, max_num=1):
         self.endTask()
-        _print = get_color_print()
-        self.startTask(_("Installing %s")%
-                       _print.foreground(Colors.YELLOW).bold(str(pkg)))
+        _print = self.color_print
+        if max_num > 1:
+            one = _print.foreground(Colors.YELLOW).bold("{0}", num)
+            two = _print.foreground(Colors.YELLOW).bold("{0}", max_num)
+            part = " (%s of %s)" % (one, two)
+        else:
+            part = ""
+        _print = _print.foreground(Colors.RED)
+
+        self.startTask(_("Unmerging%s %s") % (part, _print.bold(str(pkg))))
+
+    def emergelike(self, cmd, *params):
+        cmd_path = getProgPath(cmd)
+        if not cmd_path:
+            raise UpdateError(_("Failed to find %s command") % cmd)
+        with EmergeParser(
+                emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
+            emerge.emerging.add_observer(self._printEmergePackage)
+            emerge.installing.add_observer(self._printInstallPackage)
+            emerge.uninstalling.add_observer(self._printUninstallPackage)
+            try:
+                emerge.run()
+            except EmergeError:
+                self.printPre(self._emerge_translate(emerge.emerging_error.log))
+                raise
+        return True
+
+    def _display_pretty_package_list(self, pkglist, remove_list=False):
+        """
+        Отобразить список пакетов в "удобночитаемом" виде
+        """
+        _print = self.color_print
+        ebuild_color = TextState.Colors.GREEN
+        binary_color = TextState.Colors.PURPLE
+        remove_color = TextState.Colors.LIGHT_RED
+        for pkg in sorted([PackageInformation.add_info(x) for x in
+                           pkglist],
+                          key=lambda y: y['CATEGORY/PN']):
+            if remove_list:
+                pkgcolor = _print.foreground(remove_color)
+            else:
+                if pkg['binary']:
+                    pkgcolor = _print.foreground(binary_color)
+                else:
+                    pkgcolor = _print.foreground(ebuild_color)
+            fullname = _(pkg.info['DESCRIPTION']).capitalize()
+            shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
+            if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB":
+                size = " (%s)" % pkg["SIZE"]
+            else:
+                size = ""
+            self.printDefault("   - {fullname} {shortname}{size}".format(
+                fullname=fullname, shortname=shortname, size=size))
+
+    def _display_install_package(self, emerge):
+        """
+        Отобразить список устанавливаемых пакетов
+        """
+        # подробный список пакетов
+        if self.clVars.Get('cl_verbose_set') == 'on':
+            self.printPre(str(emerge.install_packages))
+        else:
+            _print = self.color_print
+            pkglist = emerge.install_packages.list
+            self.printSUCCESS(_print.bold(
+                _("List packages for installation")))
+            self._display_pretty_package_list(pkglist)
+            # TODO: список удаляемых пакетов во время установки
+        if str(emerge.download_size):
+            self.printSUCCESS(_print.bold(
+                _("{size} will be downloaded").format(
+                    size=emerge.download_size)))
+
+    def _display_remove_list(self, emerge):
+        """
+        Отобразить список удаляемых пакетов
+        """
+        # подробный список пакетов
+        if self.clVars.Get('cl_verbose_set') == 'on':
+            self.printPre(str(emerge.uninstall_packages))
+        else:
+            _print = self.color_print
+            pkglist = emerge.uninstall_packages.list
+            self.printSUCCESS(_print.bold(
+                _("List removal packages")))
+            self._display_pretty_package_list(pkglist, remove_list=True)
+
+    def getCacheOnWorld(self, params, packages, check=False):
+        if "@world" in packages:
+            from calculate.update.utils.cl_update import ClUpdateAction
+            elog = EmergeLog(
+                EmergeLogNamedTask(ClUpdateAction.log_names['premerge']))
+            if check and (elog.list or elog.remove_list):
+                self.emerge_cache.drop_cache()
+                return params, packages
+            installed_pkgs = elog.list
+            new_packages = self.emerge_cache.get_cached_package_list()
+            if new_packages is not None:
+                return "-1O", ["=%s" % x for x in new_packages
+                               if not str(x) in installed_pkgs]
+        return params, packages
+
+    def updateCache(self, pkg_list):
+        self.emerge_cache.set_cache(pkg_list)
+        from calculate.update.utils.cl_update import ClUpdateAction
+        elog = EmergeLog(
+            EmergeLogNamedTask(ClUpdateAction.log_names['premerge']))
+        elog.mark_end_task()
+
+    def premerge(self, param, *packages):
+        """
+        Вывести информацию об обновлении
+        """
+
+        class MockEmergeCommand(EmergeCommand):
+            """
+            Заглушка, для выполнения команд
+            """
+
+            def __init__(self, packages, *args, **kwargs):
+                EmergeCommand.__init__(self, packages, *args, **kwargs)
+
+            def execute(self):
+                if self.child is None:
+                    filename = '/tmp/mylog.log'
+                    self.child = pexpect.spawn("/bin/cat",
+                                               [filename], maxread=20000, searchwindowsize=10000)
+                    if not path.exists(filename):
+                        raise EmergeError(_("File %s not found" % filename))
+                return self.child
+
+        param, packages = self.getCacheOnWorld(param, packages)
+        param = [param, "-pv"]
+        #print "PREMERGE",packages,param
+
+        if not packages:
+            self.printSUCCESS(_("The system is up to date"))
+            return True
+        with EmergeParser(EmergeCommand(list(packages),
+                                          extra_params=param)) as emerge:
+            try:
+                emerge.run()
+                if "@world" in packages:
+                    if emerge.install_packages.remove_list:
+                        self.emerge_cache.drop_cache()
+                    else:
+                        self.updateCache(emerge.install_packages.list)
+                if not emerge.install_packages.list:
+                    self.printSUCCESS(_("The system is up to date"))
+                    return True
+                self._display_install_package(emerge)
+            except EmergeError:
+                self.emerge_cache.drop_cache()
+                self.printPre(self._emerge_translate(emerge.prepare_error))
+                raise
+            return self.askConfirm(
+                _("Would you like to merge these packages?"), "yes")
+        return True
+
+    def _emerge_translate(self, s):
+        return RegexpLocalization('cl_emerge').translate(str(s))
 
     def emerge(self, param, *packages):
         """
         Выполнить сборку пакета
         """
-        with Emerge(list(packages), extra_params=[param]) as emerge:
+        #TODO: проверить ошибку при depclean
+        class MockEmergeCommand(EmergeCommand):
+            """
+            Заглушка, для выполнения команд
+            """
+
+            def __init__(self, packages, *args, **kwargs):
+                EmergeCommand.__init__(self, packages, *args, **kwargs)
+
+            def execute(self):
+                filename = '/tmp/ppp.log'
+                if self.child is None:
+                    self.child = pexpect.spawn("/bin/cat",
+                                               #['/tmp/emerge.noupdate'])
+                                               [filename])
+                if not path.exists(filename):
+                    raise EmergeError(_("File %s not found" % filename))
+                return self.child
+
+        param, packages = self.getCacheOnWorld(param, packages, check=True)
+        #print "EMERGE",packages,param
+        ask_emerge = self.clVars.Get('cl_update_precheck_set') == 'off'
+        with EmergeParser(EmergeCommand(list(packages),
+                                        extra_params=[param])) as emerge:
             try:
-                update_list = emerge.get_update_list()
-                if not update_list:
-                    self.printSUCCESS(_("Nothing to merge"))
+                emerge.question.action = lambda x: False
+                emerge.run()
+                if not emerge.install_packages.list:
+                    #self.printSUCCESS(_("Nothing to merge"))
                     return True
-                self.printPre(update_list)
+                if ask_emerge:
+                    self.printPre(str(emerge.install_packages))
+            except EmergeError:
+                self.printPre(self._emerge_translate(emerge.prepare_error))
+                raise
+            if (ask_emerge and self.askConfirm(
+                    _("Would you like to merge these packages?")) == 'no'):
+                raise KeyboardInterrupt
+            emerge.command.send("yes\n")
+
+            emerge.emerging.add_observer(self._printEmergePackage)
+            emerge.installing.add_observer(self._printInstallPackage)
+            emerge.uninstalling.add_observer(self._printUninstallPackage)
+            try:
+                emerge.run()
             except EmergeError as e:
-                if e.errno == EmergeError.CONFLICT:
-                    self.printPre(emerge.update_block)
-                    self.printPre(emerge.conflict_block)
-                elif e.errno == EmergeError.CUSTOM:
-                    self.printPre(emerge.custom_error)
+                self.printPre(self._emerge_translate(emerge.emerging_error.log))
+                raise
+        return True
+
+    def depclean(self):
+        """
+        Выполнить очистку системы от лишних пакетов
+        """
+        with EmergeParser(EmergeCommand(["--depclean"])) as emerge:
+            try:
+                emerge.question.action = lambda x: False
+                emerge.run()
+                if not emerge.uninstall_packages.list:
+                    return True
+                self._display_remove_list(emerge)
+            except EmergeError:
+                self.printPre(self._emerge_translate(emerge.prepare_error))
                 raise
             if (self.askConfirm(
-                    _("Would you like to merge these packages?")) == 'no'):
+                    _("Would you like to unmerge these packages?")) == 'no'):
                 return False
+            emerge.command.send("yes\n")
 
-
-            emerge.handle_emerging(self._printEmergePackage)
-            emerge.handle_installing(self._printInstallPackage)
+            emerge.uninstalling.add_observer(self._printUninstallPackage)
             try:
-                return emerge.install()
+                emerge.run()
             except EmergeError:
-                self.printPre(
-                    convert_console_to_xml(readFile(emerge.get_error_log())))
+                self.printPre(self._emerge_translate(emerge.emerging_error.log))
                 raise
+        return True
+
+    def update_task(self, task_name):
+        """
+        Декоратор для добавления меток запуска и останова задачи
+        """
+
+        def decor(f):
+            def wrapper(self, *args, **kwargs):
+                logger = EmergeLog(EmergeLogNamedTask(task_name))
+                logger.mark_begin_task()
+                ret = f(self, *args, **kwargs)
+                if ret:
+                    logger.mark_end_task()
+                return ret
+
+            return wrapper
+
+        return decor
+

update/utils/cl_update.py

@@ -17,9 +17,12 @@
 import sys
 from calculate.core.server.func import Action, Tasks
 from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
+from calculate.lib.utils.colortext import get_color_print, Colors
 from calculate.lib.utils.files import FilesError
-from calculate.update.update import UpdateError, EmergeError
-from calculate.update.package_tools import GitError, Emerge
+from calculate.update.update import UpdateError
+from calculate.update.emerge_parser import EmergeError
+from calculate.update.package_tools import GitError, Eix, EmergeLog, \
+    EmergeLogNamedTask, PackageList
 
 setLocalTranslate('cl_update3', sys.modules[__name__])
 __ = getLazyLocalTranslate(_)
@@ -36,81 +39,238 @@ class ClUpdateAction(Action):
     failedMessage = __("Update failed")
     interruptMessage = __("Update manually interrupted")
 
+    def was_installed(pkg, task_name):
+        def func():
+            task = EmergeLog(EmergeLogNamedTask(task_name))
+            return bool(PackageList(task.list)[pkg])
+        return func
+
+    def need_upgrade(pkg):
+        def func():
+            return bool(Eix(pkg, Eix.Option.Upgrade).get_packages())
+        return func
+
+    def pkg_color(text):
+        _print = get_color_print()
+        return _print.bold.foreground(Colors.BLUE)(text)
+
+    log_names = {'premerge': "check updates",
+                 'python_updater': "update python modules",
+                 'perl_cleaner': "update perl modules",
+                 'kernel_modules': "update kernel modules",
+                 'xorg_modules': "update xorg modules",
+                 'preserved_libs': "update preserved libs",
+                 'revdep': "revdep rebuild"}
+
     emerge_tasks = [
-        {'name': 'update_portage',
-         'message': 'Update portage',
-         'method': 'Update.emerge("-u","portage")',
-         'condition': lambda : Emerge('-uOp','portage').has_update('sys-apps/portage')
+        {'name': 'premerge_group',
+         'group': __("Checking updates"),
+         'tasks': [
+            {'name': 'premerge',
+             'message': __("Calculating dependencies"),
+             'method': 'Update.premerge("-uDN","--with-bdeps=y","@world")',
+            }],
         },
-        {'name': 'update_python',
-         'message': 'Update python',
-         'method': 'Update.emerge("-u","python")',
-         'condition': lambda : Emerge('-uOp','python').has_update('sys-apps/portage') >= "3.0"
+        {'name': 'update',
+         'depend': (Tasks.result("premerge", eq='yes') |
+                    Tasks.hasnot("premerge"))
         },
-
-        {'name': 'update_world',
-         'message': __("Emerge world"),
-         'method': 'Update.emerge("-uDN","@world")',
-         }
-    ]
-
-    # список задач для дейсвия
-    tasks = [
-        {'name': 'sync_reps',
-         'foreach': 'cl_update_sync_rep',
-         'message': __("Syncing <b>{eachvar}</b> repository"),
-         'method': 'Update.syncRepositories(eachvar)',
-         'condition': lambda Get: Get('cl_update_sync_rep')
+        {'name': 'update:update_portage',
+         'group': __("Updating portage"),
+         'tasks': [
+             {'name': 'update:update_portage_pkg',
+              'message': __("Updating {0}").format(
+                  pkg_color("sys-apps/portage")),
+              'method': 'Update.emerge("-u","portage")',
+              'condition': need_upgrade('sys-apps/portage$')
+             },
+         ]
         },
-        {'name': 'sync_other_reps',
-         'foreach': 'cl_update_other_rep_name',
-         'message': __("Syncing {eachvar} repository"),
-         'method': 'Update.syncLaymanRepository(eachvar)',
-         'condition': lambda Get: Get('cl_update_other_set') == 'on'
+        {'name': 'update:update_python',
+         'group': __("Updating python"),
+         'tasks': [
+             {'name': 'update:update_python_pkg',
+              'message': __('Updating {0}').format(
+                  pkg_color('dev-lang/python')),
+              'method': 'Update.emerge("-u","dev-lang/python")',
+              'condition': need_upgrade('dev-lang/python$')
+             },
+             {'name': 'update:python_updater',
+              'message': __('Updating python modules'),
+              'method': 'Update.emergelike("python-updater")',
+              'condition': was_installed('dev-lang/python$',
+                                         log_names['python_updater']),
+              'decoration': 'Update.update_task("%s")' % log_names[
+                  'python_updater']
+             },
+         ]
         },
-        {'name': 'sync_reps:regen_cache',
-         'foreach': 'cl_update_sync_overlay_rep',
-         'message': __("Updating cache {eachvar} repository"),
-         'essential': False,
-         'method': 'Update.regenCache(eachvar)',
-         'condition': lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                                   Get('cl_update_metadata_force') != 'skip' or
-                                   Get('cl_update_metadata_force') == 'force')
+        {'name': 'update:update_perl',
+         'group': __("Updating perl"),
+         'tasks': [
+             {'name': 'update:update_perl_pkg',
+              'message': __('Updating {0}').format(pkg_color('dev-lang/perl')),
+              'method': 'Update.emerge("-u","dev-lang/perl")',
+              'condition': need_upgrade('dev-lang/perl$')
+             },
+             {'name': 'update:perl_cleaner',
+              'message': __('Updating perl modules'),
+              'method': 'Update.emergelike("perl-cleaner", "all")',
+              'condition': was_installed('dev-lang/perl$',
+                                         log_names['perl_cleaner']),
+              'decoration': 'Update.update_task("%s")' % log_names[
+                  'perl_cleaner']
+             },
+         ]
         },
-        {'name': 'sync_other_reps:regen_other_cache',
-         'foreach': 'cl_update_other_rep_name',
-         'message': __("Updating cache {eachvar} repository"),
-         'method': 'Update.regenCache(eachvar)',
-         'essential': False,
+        {'name': 'update:update_calculate',
+         'group': __("Updating Calculate Utilities"),
+         'tasks': [
+             {'name': 'update:update_calculate_pkgs',
+              'message': __("Updating {0}").format(
+                  pkg_color("sys-apps/calculate-utilities")),
+              'method': 'Update.emerge("-u","sys-apps/calculate-utilities")',
+              'condition': need_upgrade('sys-apps/calculate-utilities$')
+             },
+         ]
         },
-        {'name': 'emerge_metadata',
-         'message': __("Metadata trasfering"),
-         'method': 'Update.emergeMetadata()',
-         'condition': lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                                   Get('cl_update_metadata_force') != 'skip' or
-                                   Get('cl_update_metadata_force') == 'force')
+        {'name': 'update:update_world',
+         'group': __("Updating all packages"),
+         'tasks': [
+             {'name': 'update:update_world',
+              'message': __("Updating {0}").format(pkg_color("world")),
+              'method': 'Update.emerge("-uDN","--with-bdeps=y","@world")',
+             },
+         ]
         },
-        {'name': 'eix_update',
-         'message': __("Updating eix cache"),
-         'method': 'Update.eixUpdate()',
-         'condition': lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                                   Get('cl_update_eixupdate_force') != 'skip' or
-                                   Get('cl_update_eixupdate_force') == 'force')
+        {'name': 'update:update_world',
+         'group': __("Cleaning system from needless packages"),
+         'tasks': [
+             {'name': 'update:update_depclean',
+              'message': __("Emerge depclean"),
+              'method': 'Update.depclean()',
+             },
+         ]
         },
-        {'name': 'dispatch',
-         'method': 'Update.applyTemplates(install.cl_source,'
-                   'cl_template_clt_set,True,None)',
-         'condition': lambda Get: (Get('cl_update_rev_set') == 'on' or
-                                   Get('cl_rebuild_world_set') == 'on')
+        {'name': 'update:update_modules',
+         'group': __("Rebuilding dependent modules"),
+         'tasks': [
+             {'name': 'update:module_rebuild',
+              'message': __('Updating kernel modules'),
+              'method': 'Update.emerge("@module-rebuild")',
+              'condition': was_installed('sys-kernel/.*source',
+                                         log_names['kernel_modules']),
+              'decoration': 'Update.update_task("%s")' % log_names[
+                  'kernel_modules']
+             },
+             {'name': 'update:x11_module_rebuild',
+              'message': __('Updating xorg-server modules'),
+              'method': 'Update.emerge("@x11-module-rebuild")',
+              'condition': was_installed('x11-base/xorg-server',
+                                         log_names['xorg_modules']),
+              'decoration': 'Update.update_task("%s")' % log_names[
+                  'xorg_modules']
+             },
+             {'name': 'update:preserved_rebuild',
+              'message': __('Updating preserved libraries'),
+              'method': 'Update.emerge("@preserved-rebuild")',
+              'condition': was_installed('.*', log_names['preserved_libs']),
+              'decoration': 'Update.update_task("%s")' % log_names[
+                  'preserved_libs']
+             },
+             {'name': 'update:revdev_rebuild',
+              'message': __('Executing {0}').format("revdep-rebuild"),
+              'method': 'Update.emergelike("revdep-rebuild")',
+              'condition': was_installed('.*', log_names['revdep']),
+              'decoration': 'Update.update_task("%s")' % log_names['revdep']
+             }
+         ]
         },
-        # сообщение удачного завершения при обновлении репозиториев
-        {'name': 'success_syncrep',
-         'message': __("<invert>Synchronization</invert> finished!"),
-         'depend': (Tasks.success() & Tasks.has_any("sync_reps",
-                                                    "sync_other_reps",
-                                                    "emerge_metadata",
-                                                    "eix_update")),
-        }] + emerge_tasks + [
+    ]
+
+    # список задач для дейсвия
+    tasks = [
+        {'name': 'reps_synchronization',
+         'group': __("Repository synchronization"),
+         'tasks': [
+             {'name': 'sync_reps',
+              'foreach': 'cl_update_sync_rep',
+              'message': __("Syncing {0} repository").format(
+                  pkg_color("{{eachvar}}")),
+              'method': 'Update.syncRepositories(eachvar)',
+              'condition': lambda Get: Get('cl_update_sync_rep')
+             },
+             {'name': 'sync_other_reps',
+              'foreach': 'cl_update_other_rep_name',
+              'message': __("Syncing {0} repository").format(
+                  pkg_color("{{eachvar}}")),
+              'method': 'Update.syncLaymanRepository(eachvar)',
+              'condition': lambda Get: Get('cl_update_other_set') == 'on'
+             },
+             {'name': 'sync_reps:regen_cache',
+              'foreach': 'cl_update_sync_overlay_rep',
+              'message': __("Updating cache {0} repository").format(
+                  pkg_color("{{eachvar}}")),
+              'essential': False,
+              'method': 'Update.regenCache(eachvar)',
+              'condition': (
+                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
+                               Get('cl_update_metadata_force') != 'skip' or
+                               Get('cl_update_metadata_force') == 'force'))
+             },
+             {'name': 'sync_other_reps:regen_other_cache',
+              'foreach': 'cl_update_other_rep_name',
+              'message': __("Updating cache {0} repository").format(
+                  pkg_color("{{eachvar}}")),
+              'method': 'Update.regenCache(eachvar)',
+              'essential': False,
+             },
+             {'name': 'emerge_metadata',
+              'message': __("Metadata trasfering"),
+              'method': 'Update.emergeMetadata()',
+              'condition': (
+                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
+                               Get('cl_update_metadata_force') != 'skip' or
+                               Get('cl_update_metadata_force') == 'force'))
+             },
+             {'name': 'eix_update',
+              'message': __("Updating eix cache"),
+              'method': 'Update.eixUpdate()',
+              'condition': (
+                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
+                               Get('cl_update_eixupdate_force') != 'skip' or
+                               Get('cl_update_eixupdate_force') == 'force'))
+             },
+             # сообщение удачного завершения при обновлении репозиториев
+             {'name': 'success_syncrep',
+              'message': __("Synchronization finished"),
+              'depend': (Tasks.success() & Tasks.has_any("sync_reps",
+                                                         "sync_other_reps",
+                                                         "emerge_metadata",
+                                                         "eix_update")),
+             }
+         ]
+        },
+        {'name': 'reps_synchronization',
+         'group': __("System configuration"),
+         'tasks': [
+            {'name': 'revision',
+             'message': __("Fixing settings"),
+             'method': 'Update.applyTemplates(install.cl_source,'
+                       'cl_template_clt_set,True,None)',
+             'condition': lambda Get: (Get('cl_update_rev_set') == 'on' or
+                                       Get('cl_rebuild_world_set') == 'on')
+            },
+            {'name': 'world',
+             'message': __("Update system packages list"),
+             'method': 'Update.applyTemplates(install.cl_source,'
+                       'cl_template_clt_set,True,None)',
+             'condition': lambda Get: (Get('cl_update_rev_set') == 'on' or
+                                       Get('cl_rebuild_world_set') == 'on')
+            },
+         ]
+        }
+    ] + emerge_tasks + [
         # сообщение удачного завершения при обновлении ревизии
         {'name': 'success_rev',
          'message': __("System update finished!"),
@@ -120,4 +280,5 @@ class ClUpdateAction(Action):
         {'name': 'success_world',
          'message': __("World rebuild finished!"),
          'condition': lambda Get: Get('cl_rebuild_world_set') == 'on'
-        }]
+        }
+    ]

update/variables/update.py

@@ -24,6 +24,8 @@ from calculate.lib.utils.portage import searchProfile
 from calculate.lib.utils.files import readLinesFile, readFile
 
 from calculate.lib.cl_lang import setLocalTranslate
+from calculate.update.emerge_parser import EmergeCache
+
 setLocalTranslate('cl_update3',sys.modules[__name__])
 
 class VariableAcUpdateSync(ReadonlyVariable):
@@ -210,6 +212,7 @@ class VariableClUpdateBranchRep(ReadonlyVariable):
     def get(self):
         return self.Get('cl_update_rep_name')
 
+
 class VariableClUpdateBranchName(Variable):
     """
     Список доступных репозиторием
@@ -220,15 +223,17 @@ class VariableClUpdateBranchName(Variable):
         self.label = _("Branch")
 
     def choice(self):
-        return ["master","develop","update"]
+        return ["master", "develop", "update"]
 
     def get(self):
         def generateBranch():
             for reppath in self.Get('cl_update_rep_path'):
-                headPath = path.join(reppath,".git/HEAD")
+                headPath = path.join(reppath, ".git/HEAD")
                 yield readFile(headPath).rpartition('/')[2].strip() or "master"
+
         return list(generateBranch())
 
+
 class VariableClUpdateSyncRep(Variable):
     """
     Обновляемый репозиторий
@@ -380,3 +385,15 @@ class VariableClUpdateLaymanMake(Variable):
     """
     # TODO: извлечь из layman.cfg
     value = "/var/lib/layman/make.conf"
+
+class VariableClUpdatePrecheckSet(Variable):
+    """
+    Запустить предварительную проверку на обновления
+    """
+    type = "bool"
+    value = "off"
+    opt = ["--pre-check"]
+
+    def init(self):
+        self.help = _("Run pre-check updates")
+        self.label = _("run pre-check updates")

update/wsdl_update.py

@@ -39,7 +39,7 @@ class Wsdl(WsdlBase):
             # категория метода
             'category': __('Update'),
             # заголовок метода
-            'title': __("Update system"),
+            'title': __("Update System"),
             # иконка для графической консоли
             'image': 'software-properties,preferences-desktop',
             # метод присутствует в графической консоли
@@ -67,6 +67,7 @@ class Wsdl(WsdlBase):
                                     'cl_update_sync_rep', 'cl_update_branch',
                                     'cl_update_metadata_force',
                                     'cl_update_other_set',
+                                    'cl_update_precheck_set',
                                     'cl_update_eixupdate_force',
                                     'cl_templates_locate',
                                     'cl_verbose_set', 'cl_dispatch_conf'),