From 5c8b5438fc4b1f26c87aa921b479cfb907de0d3e Mon Sep 17 00:00:00 2001 From: Mike khiretskiy Date: Wed, 9 Apr 2014 15:42:15 +0400 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D0=BF=D0=B0=D1=80=D1=81=D0=B5=D1=80=20Emerge=20=D0=B8?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D1=87=D0=B8=D0=B5=20=D0=B2=D1=81=D0=BF?= =?UTF-8?q?=D0=BE=D0=BC=D0=BE=D0=B3=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D0=BD?= =?UTF-8?q?=D1=8B=D0=B5=20=D1=84=D0=B0=D0=B9=D0=BB=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 24 +- update/emerge_parser.py | 603 +++++++++++++++++++++++++++++++++++++ update/package_tools.py | 450 +++++++++------------------ update/update.py | 323 +++++++++++++++++--- update/utils/cl_update.py | 297 +++++++++++++----- update/variables/update.py | 21 +- update/wsdl_update.py | 3 +- 7 files changed, 1296 insertions(+), 425 deletions(-) create mode 100644 update/emerge_parser.py diff --git a/setup.py b/setup.py index 09d1394..7890277 100755 --- a/setup.py +++ b/setup.py @@ -20,21 +20,21 @@ __app__ = "calculate-update" __version__ = "3.1.8" -import os from distutils.core import setup from calculate.install_data import install_data -data_files = [('/usr/libexec/calculate',[('data/cl-git-wrapper',0755)])] +data_files = [('/usr/libexec/calculate', [('data/cl-git-wrapper', 0755)])] setup( - name = __app__, - version = __version__, - description = "Update system utilities", - author = "Calculate Ltd.", - author_email = "support@calculate.ru", - url = "http://calculate-linux.org", - license = "http://www.apache.org/licenses/LICENSE-2.0", - package_dir = {'calculate.update': "update"}, - data_files = data_files, - packages = ['calculate.update','calculate.update.utils','calculate.update.variables'], + name=__app__, + version=__version__, + description="Update system utilities", + author="Calculate Ltd.", + author_email="support@calculate.ru", + url="http://calculate-linux.org", + license="http://www.apache.org/licenses/LICENSE-2.0", + package_dir={'calculate.update': "update"}, + data_files=data_files, + packages=['calculate.update', 'calculate.update.utils', + 'calculate.update.variables'], cmdclass={'install_data': install_data}) diff --git a/update/emerge_parser.py b/update/emerge_parser.py new file mode 100644 index 0000000..c19450f --- /dev/null +++ b/update/emerge_parser.py @@ -0,0 +1,603 @@ +#-*- coding: utf-8 -*- + +# Copyright 2014 Calculate Ltd. http://www.calculate-linux.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os import path + +import re +import sys +from calculate.lib.utils.colortext.palette import TextState +from calculate.lib.utils.tools import ignore +from package_tools import EmergePackage, PackageList, EmergeUpdateInfo, \ + UnmergePackage, EmergeRemoveInfo, Git, GitError + +Colors = TextState.Colors +import pexpect +from calculate.lib.utils.files import getProgPath, readLinesFile, listDirectory, \ + writeFile, readFile +from calculate.lib.utils.colortext.output import XmlOutput +from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter, + XmlConverter) + +from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _ + +setLocalTranslate('cl_update3', sys.modules[__name__]) +__ = getLazyLocalTranslate(_) + + +class EmergeError(Exception): + """ + Ошибка при сборке пакетов + """ + + +class EmergeNeedRootError(EmergeError): + pass + + +class CommandExecutor(object): + """ + Запуск программы для объекта Emerge + """ + logfile = '/var/log/calculate/lastcommand.log' + + def __init__(self, cmd, params, env=None, cwd=None, logfile=None): + self.cwd = cwd + self.env = env + self.cmd = cmd + self.params = params + self.child = None + if logfile: + self.logfile = logfile + + def execute(self): + if self.child is None: + self.child = pexpect.spawn(self.cmd, + logfile=open(self.logfile, 'w'), + env=self.env, cwd=self.cwd, timeout=None) + return self.child + + def close(self): + if self.child is not None: + self.child.close() + self.child = None + + def success(self): + if self.child: + if self.child.isalive(): + self.child.wait() + return self.child.exitstatus == 0 + return False + + def failed(self): + return not self.success() + + def send(self, s): + if self.child: + self.child.send(s) + + +class EmergeCommand(CommandExecutor): + """ + Запуск emerge для последующего анализирования + """ + # параметры по умолчанию + default_params = ["-av", "--color=y", "--nospinner"] + cmd = getProgPath("/usr/bin/emerge") + + def __init__(self, packages, extra_params=None, env=None, cwd=None, + logfile=None): + extra_params = extra_params or [] + self.child = None + self.packages = packages + self.params = self.default_params + extra_params + default_env = {'CLEAN_DELAY': '0'} + default_env.update(os.environ) + self.env = env or default_env + self.cwd = cwd + if logfile: + self.logfile = logfile + + def execute(self): + if self.child is None: + self.child = pexpect.spawn(self.cmd, self.params + self.packages, + logfile=open(self.logfile, 'w'), + env=self.env, cwd=self.cwd, timeout=None) + return self.child + + +class EmergeInformationBlock(object): + _color_block = "(?:\033\[[^m]+?m)?" + _new_line = "\r*\n" + token = None + end_token = ["\n"] + re_block = None + action = None + re_match_type = type(re.match("", "")) + re_type = type(re.compile("")) + + def __init__(self, parent): + """ + :type parent: EmergeParser + """ + self.result = None + self.text_converter = parent.text_converter + self.parent = parent + self.parent.add_element(self) + self.children = [] + + def add_element(self, element): + self.children.append(element) + + def __str__(self): + if type(self.result) == self.re_match_type: + return self.result.group() + else: + return self.result or "" + + def __nonzero__(self): + return bool(self.result) + + def __len__(self): + if self.result is None: + return 0 + else: + return len(self.result) + + def __contains__(self, item): + if self.result is None: + return 0 + else: + return item in self.result + + def _get_text(self, result): + """ + Получить результат из регулярки и преобразовать его через self.converter + """ + if result: + return self.text_converter.transform(result.rstrip()) + return "" + + def get_block(self, child): + try: + token = child.match + if type(self.end_token) == self.re_type: + child.expect(self.end_token) + match = child.match.group() + else: + child.expect_exact(self.end_token) + match = child.match + self.get_data(self.re_block.search( + token + child.before + match)) + except pexpect.EOF: + child.buffer = "".join( + [x for x in (child.before, child.after, child.buffer) + if type(x) == str]) + + def get_data(self, match): + self.result = self._get_text(match.group(1)) + + +class InstallPackagesBlock(EmergeInformationBlock): + """ + Блок emerge содержащий список пакетов для установки + """ + list = PackageList([]) + remove_list = PackageList([]) + _new_line = EmergeInformationBlock._new_line + token = "\n[" + end_token = ["\r\n\r", "\n\n"] + + re_block = re.compile(r"((?:^\[.*?{nl})+)".format(nl=_new_line), + re.MULTILINE) + + def get_data(self, match): + super(InstallPackagesBlock, self).get_data(match) + list_block = XmlConverter().transform(self.result).split('\n') + self.list = PackageList(map(EmergeUpdateInfo, list_block)) + self.remove_list = PackageList(map(EmergeRemoveInfo, list_block)) + + +class UninstallPackagesBlock(EmergeInformationBlock): + """ + Блок emerge содержащий список удаляемых пакетов + """ + list = PackageList([]) + _new_line = EmergeInformationBlock._new_line + _color_block = EmergeInformationBlock._color_block + token = ["These are the packages that would be unmerged", + "Calculating removal order"] + end_token = re.compile("All selected packages:.*\n") + re_block = re.compile(r"All selected packages: (.*?){nl}". + format(nl=_new_line, c=_color_block), re.DOTALL) + + def get_data(self, match): + super(UninstallPackagesBlock, self).get_data(match) + list_block = XmlConverter().transform(self.result).split() + self.list = PackageList(map(EmergePackage, list_block)) + + +class FinishEmergeGroup(EmergeInformationBlock): + """ + Блок завершения команды + """ + token = pexpect.EOF + # регуляреное выражение, определяющее содержит ли блок + # сообщения об ошибках + re_failed = re.compile( + r"Fetch instructions for \S+:|" + r"The following.*are necessary to proceed|" + r"!!! Multiple package .* slot have been pulled|" + r"no ebuilds to satisfy|" + r"Dependencies could not be completely resolved due to", + re.MULTILINE) + + def get_block(self, child): + if child.isalive(): + child.wait() + if child.exitstatus != 0 or self.re_failed.search(child.before): + self.children_get_block(child) + else: + self.result = True + + def children_get_block(self, child): + for block in self.children: + block.get_block(child) + + def children_action(self, child): + for block in (x for x in self.children if x.result and x.action): + if block.action(child) is False: + break + + def action(self, child): + self.children_action(child) + return False + + +class PrepareErrorBlock(EmergeInformationBlock): + """ + Блок информации с ошибками при получении списка устанавливаемых пакетов + """ + token = None + + re_drop = re.compile("news items need reading|" + "Use eselect news|" + "Calculating dependencies|" + "to read news items|" + "Local copy of remote index is up-to-date|" + "These are the packages that would be merged|" + "Process finished with exit code") + re_multi_empty_line = re.compile("(?:
){3,}", re.DOTALL) + re_strip_br = re.compile("^(?:
)+|(?:
)+$", re.DOTALL) + + def remove_needless_data(self, data): + return "\n".join([x for x in data.split('\n') + if not self.re_drop.search(x)]) + + def strip_br(self, data): + return self.re_strip_br.sub( + "", + self.re_multi_empty_line.sub("

", data)) + + def get_block(self, child): + self.result = self.strip_br( + self._get_text(self.remove_needless_data(child.before))) + + def action(self, child): + raise EmergeError(_("Emerge failed")) + + +class DownloadSizeBlock(EmergeInformationBlock): + """ + Размер скачиваемых обновлений + """ + token = "Size of downloads:" + + re_block = re.compile(r"Size of downloads:\s(\S+\s\S+)") + + def __str__(self): + if self.result: + return self.result + else: + return "0 kB" + + +class QuestionBlock(EmergeInformationBlock): + """ + Блок вопроса + """ + default_answer = "no" + _color_block = EmergeInformationBlock._color_block + token = "Would you like" + end_token = ["]", "\n"] + re_block = re.compile( + "(Would you.*)\[{c}Yes{c}/{c}No{c}".format(c=_color_block)) + + def action(self, child): + if self.result: + child.send("%s\n" % self.default_answer) + + +class NeedRootBlock(EmergeInformationBlock): + """ + Пользователь не явеляется root + """ + token = "This action requires superuser access" + + def get_data(self, child): + self.result = True + + def action(self, child): + raise EmergeNeedRootError(_("This action requires superuser access")) + + +class NotifierInformationBlock(EmergeInformationBlock): + """ + Информационный блок поддерживающий observing + """ + def __init__(self, parent): + super(NotifierInformationBlock, self).__init__(parent) + self.observers = [] + + def get_data(self, match): + self.result = match + + def add_observer(self, f): + self.observers.append(f) + + def notify(self, observer, groups): + observer(groups) + + def action(self, child): + if self.result and self.observers: + groups = self.result.groups() + for observer in self.observers: + self.notify(observer, groups) + + +class EmergingPackage(NotifierInformationBlock): + """ + Запуск устанавливаемого пакета + + ObserverFunc: (package, num=number, max_num=number, binary=binary) + """ + _color_block = EmergeInformationBlock._color_block + token = ">>> Emerging " + re_block = re.compile( + "Emerging (binary )?\({c}(\d+){c} " + "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block)) + + def notify(self, observer, groups): + observer(EmergePackage(groups[3]), num=groups[1], max_num=groups[2], + binary=bool(groups[0])) + + +class UnemergingPackage(NotifierInformationBlock): + """ + Запуск устанавливаемого пакета + + ObserverFunc: (package, num=number, max_num=number) + """ + _color_block = EmergeInformationBlock._color_block + token = ">>> Unmerging" + re_block = re.compile( + r"Unmerging (?:\({c}(\d+){c} " + r"of {c}(\d+){c}\) )?(\S+)\.\.\.".format(c=_color_block)) + + def notify(self, observer, groups): + observer(EmergePackage(groups[2]), num=groups[0], max_num=groups[1]) + + +class InstallingPackage(NotifierInformationBlock): + """ + Запуск устанавливаемого пакета + + ObserverFunc: (package, binary=binary) + """ + _color_block = EmergeInformationBlock._color_block + binary = None + + token = ">>> Installing " + re_block = re.compile( + "Installing \({c}(\d+){c} " + "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block)) + + def notify(self, observer, groups): + binary = bool(self.binary and groups[2] in self.binary) + observer(EmergePackage(groups[2]), binary=binary) + + def mark_binary(self, package): + if self.binary is None: + self.binary = [] + self.binary.append(str(package)) + + +class EmergeingErrorBlock(EmergeInformationBlock): + """ + Блок содержит информацию об ошибке во время сборки пакета + """ + token = ["* ERROR: ", " * \033[39;49;00mERROR: "] + end_token = "Working directory:" + re_block = re.compile("ERROR: (\S*) failed \([^)]+\).*?" + "The complete build log is located at '([^']+)", + re.DOTALL) + package = "" + + def get_data(self, match): + self.result = self._get_text(match.group(2).rstrip()) + self.package = match.group(1) + + @property + def log(self): + return self.text_converter.transform(readFile(self.result)) + + def action(self, child): + raise EmergeError(_("Emerge %s is failed") % self.package) + + +class EmergeParser(object): + """ + Парсер вывода emerge + """ + + def __init__(self, command, run=False): + self.text_converter = ConsoleCodes256Converter(XmlOutput()) + self.command = command + self.elements = {} + + self.install_packages = InstallPackagesBlock(self) + self.uninstall_packages = UninstallPackagesBlock(self) + self.question = QuestionBlock(self) + self.finish_block = FinishEmergeGroup(self) + self.need_root = NeedRootBlock(self) + self.prepare_error = PrepareErrorBlock(self.finish_block) + self.download_size = DownloadSizeBlock(self) + self.emerging_error = EmergeingErrorBlock(self) + + self.installing = InstallingPackage(self) + self.uninstalling = UnemergingPackage(self) + self.emerging = EmergingPackage(self) + + self.emerging.add_observer(self.mark_binary) + if run: + self.run() + + def mark_binary(self, package, binary=False, **kw): + if binary: + self.installing.mark_binary(package) + + def add_element(self, element): + if element.token: + if type(element.token) == list: + for token in element.token: + self.elements[token] = element + else: + self.elements[element.token] = element + + def run(self): + """ + Запустить команду + """ + child = self.command.execute() + + while True: + index = child.expect_exact(self.elements.keys()) + element = self.elements.values()[index] + element.get_block(child) + if element.action: + if element.action(child) is False: + break + + def close(self): + self.command.close() + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self.close() + + +class MtimeCheckvalue(object): + def __init__(self, *fname): + self.fname = fname + + def value_func(self, fn): + return str(int(os.stat(fn).st_mtime)) + + def get_check_values(self, file_list): + for fn in file_list: + if path.exists(fn) and not path.isdir(fn): + yield fn, self.value_func(fn) + else: + for k, v in self.get_check_values( + listDirectory(fn, fullPath=True)): + yield k, v + + def checkvalues(self): + return self.get_check_values(self.fname) + + +class GitCheckvalue(object): + def __init__(self, rpath): + self.rpath = rpath + self.git = Git() + + def checkvalues(self): + with ignore(GitError): + if self.git.is_git(self.rpath): + yield self.rpath, Git().getCurrentCommit(self.rpath) + + +class EmergeCache(object): + """ + Кэш пакетов + """ + cache_file = '/var/lib/calculate/calculate-update/world.cache' + # список файлов проверяемый по mtime на изменения + check_list = [MtimeCheckvalue('/etc/make.conf', + '/etc/portage', + '/etc/make.profile', + '/var/cache/edb/binhost')] + + def __init__(self): + self.files_control_values = {} + self.pkg_list = PackageList([]) + + def set_cache(self, package_list): + """ + Установить в кэш список пакетов + """ + with writeFile(self.cache_file) as f: + for fn, val in self.get_control_values().items(): + f.write("{fn}={val}\n".format(fn=fn, val=val)) + f.write('\n') + for pkg in package_list: + f.write("%s\n"% str(pkg)) + + def drop_cache(self): + if path.exists(self.cache_file): + with ignore(OSError): + os.unlink(self.cache_file) + + def get_cached_package_list(self): + self.read_cache() + if self.check_actuality(): + return self.pkg_list + return None + + def check_actuality(self): + """ + Кэш считается актуальным если ни один из файлов не менялся + """ + return self.get_control_values() == self.files_control_values + + def read_cache(self): + self.files_control_values = {} + cache_file_lines = readLinesFile(self.cache_file) + for line in cache_file_lines: + if not "=" in line: + break + k, v = line.split('=') + self.files_control_values[k] = v.strip() + self.pkg_list = PackageList(cache_file_lines) + + def get_control_values(self): + def generate(): + for obj in self.check_list: + for checkvalue in obj.checkvalues(): + yield checkvalue + return dict(generate()) diff --git a/update/package_tools.py b/update/package_tools.py index 243e6a6..5bb08a4 100644 --- a/update/package_tools.py +++ b/update/package_tools.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import Mapping +from collections import Mapping, defaultdict import re import sys @@ -28,12 +28,10 @@ from os import path from calculate.lib.utils.files import (getProgPath, STDOUT, PercentProgress, process, readFile, readLinesFile) -from calculate.lib.utils.colortext.output import XmlOutput -from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter, - XmlConverter) from calculate.lib.utils.common import cmpVersion from contextlib import closing + import xml.etree.ElementTree as ET from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate @@ -127,7 +125,6 @@ class Layman: self._add_to_makeconf(rpath) return True - class Git: """ Объект для управление git репозиторием @@ -487,7 +484,7 @@ class EmergePackage(Mapping): pv = r"(?:-(\d[^-]*?))?" pr = r"(?:-(r\d+))?" tbz = r"(?:.(tbz2))?" - slot = r'(?::(\d+(?:\.\d+)*(?:/\d+(?:\.\d+))?))?' + slot = r'(?::(\w+(?:\.\w+)*(?:/\w+(?:\.\w+)*)?))?' repo = r'(?:::(\w+))?' reParse = re.compile( @@ -515,7 +512,10 @@ class EmergePackage(Mapping): 'REPO': x[REPO] or self.default_repo, 'CATEGORY/PN': "%s/%s" % (x[CATEGORY], x[PN]), 'PR': x[PR] or 'r0'} - d['PVR'] = "%s-%s" % (d['PV'], d['PR']) + if x[PR]: + d['PVR'] = "%s-%s" % (d['PV'], d['PR']) + else: + d['PVR'] = d['PV'] if d['PF'].endswith('-r0'): d['PF'] = d['PF'][:-3] return d.copy() @@ -538,27 +538,31 @@ class EmergePackage(Mapping): if "CATEGORY/PN" in version and "PVR" in version: if self['CATEGORY/PN'] < version['CATEGORY/PN']: return True - version = version['PVR'] - return cmpVersion(self['PVR'], version) == -1 + elif self['CATEGORY/PN'] > version['CATEGORY/PN']: + return False + version = "%s-%s" % (version['PV'], version['PR']) + currentVersion = "%s-%s" % (self['PV'], self['PR']) + return cmpVersion(currentVersion, version) == -1 def __eq__(self, version): if "CATEGORY" in version and "PF" in version: return ("%s/%s" % (self['CATEGORY'], self['PF']) == "%s/%s" % (version['CATEGORY'], version['PF'])) else: - return cmpVersion(self['PVR'], version) == 0 + currentVersion = "%s-%s" % (self['PV'], self['PR']) + return cmpVersion(currentVersion, version) == 0 def __init__(self, package): if isinstance(package, EmergePackage): self.__result = package.__result - self.__package = package.__package + self._package = package._package else: - self.__package = package + self._package = package self.__result = None def __getitem__(self, item): if not self.__result: - self.__result = self._parsePackageString(self.__package) + self.__result = self._parsePackageString(self._package) return self.__result[item] def __repr__(self): @@ -568,6 +572,85 @@ class EmergePackage(Mapping): return "%s/%s" % (self['CATEGORY'], self['PF']) +class PackageInformation: + """ + Объект позволяет получать информацию о пакете из eix + """ + eix_cmd = getProgPath("/usr/bin/eix") + + query_packages = [] + information_cache = defaultdict(dict) + + fields = ["DESCRIPTION"] + + def __init__(self, pkg): + self._pkg = pkg + if not pkg in self.query_packages: + self.query_packages.append(pkg) + + def __getitem__(self, item): + if not self._pkg['CATEGORY/PN'] in self.information_cache: + self.query_information() + try: + return self.information_cache[self._pkg['CATEGORY/PN']][item] + except KeyError: + return "" + + def query_information(self): + pkg_list = "|".join( + [x['CATEGORY/PN'].replace("+", r"\+") for x in self.query_packages]) + output = pexpect.spawn(self.eix_cmd, ["--xml", pkg_list]).read() + xml = ET.fromstring(output) + for pkg in self.query_packages: + cat_pn = pkg['CATEGORY/PN'] + if not cat_pn in self.information_cache: + descr_node = xml.find( + 'category[@name="%s"]/package[@name="%s"]/description' + % (pkg['CATEGORY'], pkg['PN'])) + if descr_node is not None: + self.information_cache[cat_pn]['DESCRIPTION'] = \ + descr_node.text + while self.query_packages: + self.query_packages.pop() + + @classmethod + def add_info(cls, pkg): + pkg.info = cls(pkg) + return pkg + +class UnmergePackage(EmergePackage): + """ + Информация об обновлении одного пакета + """ + re_pkg_info = re.compile("^\s(\S+)\n\s+selected:\s(\S+)",re.MULTILINE) + + def __init__(self, package): + super(UnmergePackage, self).__init__(package) + if not isinstance(package, EmergePackage): + self._package = self.convert_package_info(package) + + def convert_package_info(self, package): + match = self.re_pkg_info.search(package) + if match: + return "%s-%s" % match.groups() + return "" + + +def recalculate_update_info(cls): + """ + Добавить + """ + cls.update_info = re.compile( + r"^({install_info})\s+({atom_info})\s*(?:{prev_version})?" + r"\s*({use_info})?.*?({pkg_size})?$".format( + install_info=cls.install_info, + atom_info=cls.atom_info, + prev_version=cls.prev_version, + use_info=cls.use_info, + pkg_size=cls.pkg_size), re.MULTILINE) + return cls + +@recalculate_update_info @total_ordering class EmergeUpdateInfo(Mapping): """ @@ -578,27 +661,22 @@ class EmergeUpdateInfo(Mapping): atom_info = r"\S+" use_info = 'USE="[^"]+"' prev_version = "\[([^\]]+)\]" - update_info = re.compile( - r"^({install_info})\s+({atom_info})\s+(?:{prev_version})?" - r"\s*({use_info})?".format(install_info=install_info, - atom_info=atom_info, - prev_version=prev_version, - use_info=use_info)) + pkg_size = r"\d+ \w+" - attrs = ['binary', 'REPLACING_VERSIONS'] + attrs = ['binary', 'REPLACING_VERSIONS', 'SIZE'] def __init__(self, data): self._data = data self._package = None self._info = {} - def _parseData(self): r = self.update_info.search(self._data) if r: self._info['binary'] = r.group(2) == 'binary' self._package = EmergePackage(r.group(3)) self._info['REPLACING_VERSIONS'] = r.group(4) or "" + self._info['SIZE'] = r.group(6) or "" def __iter__(self): return chain(EmergePackage.attrs, self.attrs) @@ -639,16 +717,12 @@ class EmergeUpdateInfo(Mapping): def __str__(self): return "%s/%s" % (self['CATEGORY'], self['PF']) - -class EmergeError(Exception): +@recalculate_update_info +class EmergeRemoveInfo(EmergeUpdateInfo): """ - Ошибка при сборке пакетов + Информация об удалении одного пакета (в списке обновляемых пакетов) """ - (NEED_ROOT, NO_EBUILD, CONFLICT, CUSTOM) = range(0, 4) - - def __init__(self, msg, errno=CUSTOM): - Exception.__init__(self, msg) - self.errno = errno + install_info = "\[(uninstall)[^\]]+\]" class Eix: @@ -721,277 +795,7 @@ class Eix: return iter(()) -class Emerge: - """ - Объект для выполнения сборки пакетов - """ - # параметры по умолчанию - default_params = ["-av", "--color=y", "--nospinner"] - cmd = getProgPath("/usr/bin/emerge") - # шаблон выборки списка пакетов - install_pkgs_pattern = \ - "(\\[[^\\]]+\\]\\s*(\\S+).*(?:{nl}|$))".format(nl="\r*\n") - re_install_pkgs = re.compile("^%s+" % install_pkgs_pattern, - re.MULTILINE) - # конфликтный пакет - _conflict_pkg = r"[\w-]+/[\w-]+:\d+" - # конкретная версия конфликтного пакета - _conflict_candidate = r" \(.*" - # пакеты в которым необходим конфликтный пакет - _conflict_req = r" \S.*" - # шаблон конфликтного блока - conflict_block = \ - "({nl}{nl}({0}{nl}({1}{nl})+{nl})+)".format(_conflict_candidate, - _conflict_req, - nl="\r*\n") - re_conflict_block = re.compile( - "^{0}{conflict_block}([\w\W]+{conflict_block})*".format( - _conflict_pkg, conflict_block=conflict_block), - re.MULTILINE) - # шаблон устновки консольного цвета - _color_block = "(?:\033\[[^m]+?m)?" - re_custom_error = re.compile( - r"^Calculating dependencies.*{nl}".format(nl="\r*\n") + - "%s*" % install_pkgs_pattern + - r"(?P[\w\W]+)", re.MULTILINE) - - # regexp для удаления неиспользуемой информации - re_cut_blocks = \ - re.compile(r"(It may be possible to solve.*?" - "solve this conflict automatically\.|" - "For more information, see.*?Gentoo Handbook\.)\s*", re.S) - - expect_need_root = "This action requires superuser access" - expect_conflict = "Multiple package[ \w]+pulled" - expect_to_merge = "Would you like to merge" - expect_nothing_merge = ("Nothing to merge|No outdated packages|" - "\s0 packages") - expect_no_ebuild = "no ebuilds to satisfy \"" - expect_custom_question = "{c}Yes{c}/{c}No{c}".format(c=_color_block) - expect_emerge_package = ("Emerging (binary )?\({c}(\d+){c} " - "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format( - c=_color_block)) - expect_install_package = ("Installing (binary )?\({c}(\d+){c} " - "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format( - c=_color_block)) - expect_error_on_install = "ERROR: \S* failed \([^)]+\)" - - def __init__(self, packages, extra_params=[], env=None, cwd=None, - command_runner=None, - converter=ConsoleCodes256Converter(XmlOutput())): - self.text_converter = converter - self.packages = packages - self.params = self.default_params + extra_params - self.child = None - self.update_block = None - self.conflict_block = None - self.update_packages = None - self.custom_error = "" - self.env = env - self.cwd = cwd - # событие начало сборки - self.event_emerging = [] - # событие начало установки - self.event_installing = [] - self.error_log = None - - def handle_emerging(self, f): - """ - Добавить обработку начала сборки пакета - """ - if not f in self.event_emerging: - self.event_emerging.append(f) - - def handle_installing(self, f): - """ - Добавить обработку начала установки пакета - """ - if not f in self.event_installing: - self.event_installing.append(f) - - def _get_text(self, re_result, already_text=False): - """ - Получить результат из регулярки и преобразовать его через self.converter - """ - if re_result: - if not already_text: - re_result = re_result.group() - result = self.re_cut_blocks.sub("", re_result) - return self.text_converter.transform(result.strip()) - return "" - - def _get_custom_error_output(self, buf): - """ - Получить вывод об прочих ошибках в ходе вычисления зависимостей - """ - r = self.re_custom_error.search(buf) - if r: - return self._get_text(r.groupdict()["result"], already_text=True) - return "" - - def _get_package_list_output(self, buf): - """ - Получить из вывода emerge часть текста со списком пакетов - """ - return self._get_text(self.re_install_pkgs.search(buf)) - - def _get_conflict_list_output(self, buf): - """ - Разобрать вывод о конфликтах среди пакетов - """ - return self._get_text(self.re_conflict_block.search(buf)) - - def get_update_block(self): - """ - Получить список устанавливаемых пакетов - """ - if self.update_block is None: - self.execute() - events = {v: k for k, v in enumerate(sorted([ - self.expect_need_root, - self.expect_conflict, - self.expect_to_merge, - self.expect_nothing_merge, - self.expect_no_ebuild, - self.expect_custom_question, - pexpect.EOF]))} - res = self.child.expect(sorted(events.keys())) - if res == events[self.expect_need_root]: - self.process_need_root() - elif res == events[self.expect_to_merge]: - self.update_block = ( - self._get_package_list_output(self.child.before)) - elif res == events[self.expect_nothing_merge]: - self.update_block = "" - elif res == events[self.expect_no_ebuild]: - raise EmergeError(_("No ebuilds to satisfy %s") % - self.child.buffer.split('"', 1)[0], - errno=EmergeError.NO_EBUILD) - elif res == events[self.expect_conflict]: - self.update_block = \ - self._get_package_list_output(self.child.before) - self.conflict_block = \ - self._get_conflict_list_output(self.child.read()) - raise EmergeError(_("Multiple package instances within " - "a single package slot"), - errno=EmergeError.CONFLICT) - elif res == events[self.expect_custom_question]: - self.custom_error = self._get_custom_error_output( - self.child.before) - self.child.sendline("no") - raise EmergeError(_("Unexpected question")) - elif res == events[pexpect.EOF]: - if re.search("Total:\s+\d+ package", self.child.before): - self.update_block = ( - self._get_package_list_output(self.child.before)) - else: - self.custom_error = self._get_custom_error_output( - self.child.before) - raise EmergeError(_("Failed to emerge")) - return self.update_block - - def notifyEmergingPackage(self, package, num=1, max_num=2, binary=False): - [f(EmergePackage(package), num=num, max_num=max_num, binary=binary) - for f in self.event_emerging] - - def notifyInstallingPackage(self, package): - [f(EmergePackage(package)) for f in self.event_installing] - - def install(self): - """ - Install packages - """ - self.execute() - self.child.sendline("yes") - pkg_name = "" - events = {v: k for k, v in enumerate(sorted([ - self.expect_need_root, - self.expect_emerge_package, - self.expect_install_package, - self.expect_error_on_install, - pexpect.EOF]))} - while True: - res = self.child.expect(sorted(events.keys())) - if res == events[self.expect_need_root]: - self.process_need_root() - elif res in (events[self.expect_install_package], - events[self.expect_emerge_package]): - pkg_name = self.child.match.group(4).strip() - if res == events[self.expect_emerge_package]: - binary_pkg = bool(self.child.match.group(1)) - current_pkg_num = int(self.child.match.group(2)) - max_pkg_num = int(self.child.match.group(3)) - self.notifyEmergingPackage(pkg_name, num=current_pkg_num, - max_num=max_pkg_num, - binary=binary_pkg) - else: - self.notifyInstallingPackage(pkg_name) - elif res == events[self.expect_error_on_install]: - if self.child.expect( - ["The complete build log is located at '[^']+", - pexpect.EOF]) == 0: - self.error_log = self.child.after.rpartition("'")[-1] - raise EmergeError(_("Emerge %s is failed") % pkg_name) - else: - if self.child.isalive(): - self.child.wait() - return self.child.exitstatus == 0 - - def execute(self): - # TODO: убрать лог в /tmp - if self.child is None: - self.child = pexpect.spawn(self.cmd, self.params + self.packages, - logfile=open('/tmp/emerge.log', 'w'), - env=self.env, cwd=self.cwd, timeout=None) - return self.child - - def close(self): - if self.child is not None: - self.child.close() - self.child = None - - def get_error_log(self): - """ - Получить путь до журнала сборки - """ - return self.error_log - - def process_need_root(self): - raise EmergeError( - _("This action requires superuser access"), - errno=EmergeError.NEED_ROOT) - - def __enter__(self): - return self - - def __exit__(self, *exc_info): - self.close() - - def get_update_list(self): - """ - Проверить есть ли в обновлениях пакет совпадающий с pkg_pattern - полученный пакет можно проверить указанной check функцией - """ - if self.update_packages is None: - list_block = XmlConverter().transform(self.get_update_block()) - self.update_packages = self._parse_update_list(list_block) - return PackageList(self.update_packages) - - def _parse_update_list(self, list_block): - return filter(lambda x: x['PN'] is not None, - map(EmergeUpdateInfo, - list_block.split('\n'))) - - -class RevdepRebuild(Emerge): - def __init__(self): - pass - - def execute(self): - pass - class EmergeLogTask(object): - def has_marker(self, line): """ Определить есть ли в строке маркер задачи @@ -1010,6 +814,7 @@ class EmergeLogTask(object): """ return "" + class EmergeLogNamedTask(EmergeLogTask): date_format = "%b %d, %Y %T" @@ -1036,19 +841,23 @@ class EmergeLogNamedTask(EmergeLogTask): """ return "*** Finished %s" % self.taskname + class EmergeLog: """ EmergeLog(after).get_update(package_pattern) """ emerge_log = "/var/log/emerge.log" - re_complete_emerge = re.compile(r":::\scompleted emerge \(.*?\) (\S+)", + re_complete_emerge = re.compile(r":::\scompleted (emerge) \(.*?\) (\S+)", re.M) + re_complete_unmerge = re.compile(r">>>\s(unmerge) success: (\S+)", re.M) def __init__(self, emerge_task=EmergeLogTask()): """ @type emerge_task: EmergeLogTask """ self.emerge_task = emerge_task + self._list = None + self._remove_list = None def _get_last_changes(self): """ @@ -1060,16 +869,36 @@ class EmergeLog: log_data.save() return log_data.restore() + @property + def list(self): + if self._list is None: + self.get_packages() + return self._list + + @property + def remove_list(self): + if self._remove_list is None: + self.get_packages() + return self._remove_list + def get_packages(self): """ Получить список пакетов """ - return list(self._parse_log(self._get_last_changes())) + self._list, self._remove_list = \ + zip(*self._parse_log(self._get_last_changes())) + self._list = filter(None,self._list) + self._remove_list = filter(None, self._remove_list) def _parse_log(self, data): - for re_match in ifilter(None, - imap(self.re_complete_emerge.search, data)): - yield re_match.group(1) + searcher = lambda x:(self.re_complete_emerge.search(x) or + self.re_complete_unmerge.search(x)) + for re_match in ifilter(None, imap(searcher, data)): + if re_match.group(1) == "emerge": + yield re_match.group(2), None + else: + yield None, re_match.group(2) + yield None, None def _set_marker(self, text_marker): with open(self.emerge_log, 'a') as f: @@ -1096,13 +925,18 @@ class PackageList(object): """ Список пакетов с возможностью среза и сравнением с версией """ + def __init__(self, packages): self._raw_list = packages + self.result = None def _packages(self): - return ifilter(lambda x: x['PN'] is not None, - imap(lambda x: x if isinstance(x, Mapping) else EmergePackage(x), - self._raw_list)) + if self.result is None: + self.result = filter(lambda x: x['PN'], + imap(lambda x: (x if isinstance(x, Mapping) + else EmergePackage(x)), + self._raw_list)) + return self.result def __getitem__(self, item): re_item = re.compile(item) @@ -1113,7 +947,7 @@ class PackageList(object): return iter(self._packages()) def __len__(self): - return len(self._raw_list) + return len(list(self._packages())) def __lt__(self, other): return any(pkg < other for pkg in self._packages()) diff --git a/update/update.py b/update/update.py index 61e21db..059c3a1 100644 --- a/update/update.py +++ b/update/update.py @@ -20,25 +20,42 @@ from os import path from calculate.lib.utils.tools import AddonError from calculate.lib.utils.colortext.palette import TextState from calculate.lib.utils.colortext import get_color_print +import pexpect -from package_tools import Git, Layman, EmergeError, Emerge +from package_tools import Git, Layman,\ + EmergeLogNamedTask, EmergeLog, GitError, \ + PackageInformation Colors = TextState.Colors from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir, - PercentProgress, process, readFile) -from calculate.lib.utils.colortext import convert_console_to_xml -from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate + PercentProgress, process) +from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, + RegexpLocalization, _) +import emerge_parser +from emerge_parser import EmergeParser, EmergeCommand, EmergeError, EmergeCache setLocalTranslate('cl_update3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) + class UpdateError(AddonError): """Update Error""" + class Update: """Основной объект для выполнения действий связанных с обновлением системы """ + def init(self): + commandLog = path.join(self.clVars.Get('core.cl_log_path'), + 'lastcommand.log') + emerge_parser.CommandExecutor.logfile = commandLog + self.color_print = get_color_print() + self.emerge_cache = EmergeCache() + self.emerge_cache.check_list = ( + self.emerge_cache.check_list + + map(emerge_parser.GitCheckvalue, + self.clVars.Get('update.cl_update_rep_path'))) def _syncRepository(self, name, url, rpath, revision, branch, cb_progress=None): @@ -51,17 +68,17 @@ class Update: if not git.checkExistsRep(rpath): if revision == "last": git.cloneRepository(url, rpath, branch, - cb_progress=cb_progress) + cb_progress=cb_progress) else: git.cloneRevRepository(url, rpath, branch, revision, - cb_progress=cb_progress) + cb_progress=cb_progress) needMeta = True else: # если нужно обновиться до конкретной ревизии if revision != "last": if revision == git.getCurrentCommit(rpath): if git.getBranch(rpath) == branch: - return True + return False # получить изменения из удаленного репозитория git.fetchRepository(rpath, cb_progress=cb_progress) # если текущая ветка не соответствует нужной @@ -98,8 +115,9 @@ class Update: self.addProgress() if clean_on_error: try: - self._syncRepository(repname, url, rpath, revision, branch, - cb_progress=self.setProgress) + if not self._syncRepository(repname, url, rpath, revision, branch, + cb_progress=self.setProgress): + return "skip" return True except GitError as e: if e.addon: @@ -135,8 +153,9 @@ class Update: else: p = process(layman, "-s", repname, stderr=STDOUT) if p.failed(): - raise UpdateError(_("Failed to update repository {rname}" - ).format(rname=repname), addon=p.read()) + raise UpdateError( + _("Failed to update repository {rname}").format(rname=repname), + addon=p.read()) return True def regenCache(self, repname): @@ -208,7 +227,7 @@ class Update: def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1): self.endTask() - _print = get_color_print() + _print = self.color_print if max_num > 1: one = _print.foreground(Colors.YELLOW).bold("{0}", num) two = _print.foreground(Colors.YELLOW).bold("{0}", max_num) @@ -220,42 +239,278 @@ class Update: else: _print = _print.foreground(Colors.GREEN) - self.startTask("Emerging%s %s" % (part, _print(str(pkg)))) + self.startTask(_("Emerging%s %s") % (part, _print(str(pkg)))) + + def _printInstallPackage(self, pkg, binary=False): + self.endTask() + _print = self.color_print + if binary: + _print = _print.foreground(Colors.PURPLE) + else: + _print = _print.foreground(Colors.GREEN) + self.startTask(_("Installing %s") % + _print(str(pkg))) - def _printInstallPackage(self, pkg): + def _printUninstallPackage(self, pkg, num=1, max_num=1): self.endTask() - _print = get_color_print() - self.startTask(_("Installing %s")% - _print.foreground(Colors.YELLOW).bold(str(pkg))) + _print = self.color_print + if max_num > 1: + one = _print.foreground(Colors.YELLOW).bold("{0}", num) + two = _print.foreground(Colors.YELLOW).bold("{0}", max_num) + part = " (%s of %s)" % (one, two) + else: + part = "" + _print = _print.foreground(Colors.RED) + + self.startTask(_("Unmerging%s %s") % (part, _print.bold(str(pkg)))) + + def emergelike(self, cmd, *params): + cmd_path = getProgPath(cmd) + if not cmd_path: + raise UpdateError(_("Failed to find %s command") % cmd) + with EmergeParser( + emerge_parser.CommandExecutor(cmd_path, params)) as emerge: + emerge.emerging.add_observer(self._printEmergePackage) + emerge.installing.add_observer(self._printInstallPackage) + emerge.uninstalling.add_observer(self._printUninstallPackage) + try: + emerge.run() + except EmergeError: + self.printPre(self._emerge_translate(emerge.emerging_error.log)) + raise + return True + + def _display_pretty_package_list(self, pkglist, remove_list=False): + """ + Отобразить список пакетов в "удобночитаемом" виде + """ + _print = self.color_print + ebuild_color = TextState.Colors.GREEN + binary_color = TextState.Colors.PURPLE + remove_color = TextState.Colors.LIGHT_RED + for pkg in sorted([PackageInformation.add_info(x) for x in + pkglist], + key=lambda y: y['CATEGORY/PN']): + if remove_list: + pkgcolor = _print.foreground(remove_color) + else: + if pkg['binary']: + pkgcolor = _print.foreground(binary_color) + else: + pkgcolor = _print.foreground(ebuild_color) + fullname = _(pkg.info['DESCRIPTION']).capitalize() + shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"])) + if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB": + size = " (%s)" % pkg["SIZE"] + else: + size = "" + self.printDefault(" - {fullname} {shortname}{size}".format( + fullname=fullname, shortname=shortname, size=size)) + + def _display_install_package(self, emerge): + """ + Отобразить список устанавливаемых пакетов + """ + # подробный список пакетов + if self.clVars.Get('cl_verbose_set') == 'on': + self.printPre(str(emerge.install_packages)) + else: + _print = self.color_print + pkglist = emerge.install_packages.list + self.printSUCCESS(_print.bold( + _("List packages for installation"))) + self._display_pretty_package_list(pkglist) + # TODO: список удаляемых пакетов во время установки + if str(emerge.download_size): + self.printSUCCESS(_print.bold( + _("{size} will be downloaded").format( + size=emerge.download_size))) + + def _display_remove_list(self, emerge): + """ + Отобразить список удаляемых пакетов + """ + # подробный список пакетов + if self.clVars.Get('cl_verbose_set') == 'on': + self.printPre(str(emerge.uninstall_packages)) + else: + _print = self.color_print + pkglist = emerge.uninstall_packages.list + self.printSUCCESS(_print.bold( + _("List removal packages"))) + self._display_pretty_package_list(pkglist, remove_list=True) + + def getCacheOnWorld(self, params, packages, check=False): + if "@world" in packages: + from calculate.update.utils.cl_update import ClUpdateAction + elog = EmergeLog( + EmergeLogNamedTask(ClUpdateAction.log_names['premerge'])) + if check and (elog.list or elog.remove_list): + self.emerge_cache.drop_cache() + return params, packages + installed_pkgs = elog.list + new_packages = self.emerge_cache.get_cached_package_list() + if new_packages is not None: + return "-1O", ["=%s" % x for x in new_packages + if not str(x) in installed_pkgs] + return params, packages + + def updateCache(self, pkg_list): + self.emerge_cache.set_cache(pkg_list) + from calculate.update.utils.cl_update import ClUpdateAction + elog = EmergeLog( + EmergeLogNamedTask(ClUpdateAction.log_names['premerge'])) + elog.mark_end_task() + + def premerge(self, param, *packages): + """ + Вывести информацию об обновлении + """ + + class MockEmergeCommand(EmergeCommand): + """ + Заглушка, для выполнения команд + """ + + def __init__(self, packages, *args, **kwargs): + EmergeCommand.__init__(self, packages, *args, **kwargs) + + def execute(self): + if self.child is None: + filename = '/tmp/mylog.log' + self.child = pexpect.spawn("/bin/cat", + [filename], maxread=20000, searchwindowsize=10000) + if not path.exists(filename): + raise EmergeError(_("File %s not found" % filename)) + return self.child + + param, packages = self.getCacheOnWorld(param, packages) + param = [param, "-pv"] + #print "PREMERGE",packages,param + + if not packages: + self.printSUCCESS(_("The system is up to date")) + return True + with EmergeParser(EmergeCommand(list(packages), + extra_params=param)) as emerge: + try: + emerge.run() + if "@world" in packages: + if emerge.install_packages.remove_list: + self.emerge_cache.drop_cache() + else: + self.updateCache(emerge.install_packages.list) + if not emerge.install_packages.list: + self.printSUCCESS(_("The system is up to date")) + return True + self._display_install_package(emerge) + except EmergeError: + self.emerge_cache.drop_cache() + self.printPre(self._emerge_translate(emerge.prepare_error)) + raise + return self.askConfirm( + _("Would you like to merge these packages?"), "yes") + return True + + def _emerge_translate(self, s): + return RegexpLocalization('cl_emerge').translate(str(s)) def emerge(self, param, *packages): """ Выполнить сборку пакета """ - with Emerge(list(packages), extra_params=[param]) as emerge: + #TODO: проверить ошибку при depclean + class MockEmergeCommand(EmergeCommand): + """ + Заглушка, для выполнения команд + """ + + def __init__(self, packages, *args, **kwargs): + EmergeCommand.__init__(self, packages, *args, **kwargs) + + def execute(self): + filename = '/tmp/ppp.log' + if self.child is None: + self.child = pexpect.spawn("/bin/cat", + #['/tmp/emerge.noupdate']) + [filename]) + if not path.exists(filename): + raise EmergeError(_("File %s not found" % filename)) + return self.child + + param, packages = self.getCacheOnWorld(param, packages, check=True) + #print "EMERGE",packages,param + ask_emerge = self.clVars.Get('cl_update_precheck_set') == 'off' + with EmergeParser(EmergeCommand(list(packages), + extra_params=[param])) as emerge: try: - update_list = emerge.get_update_list() - if not update_list: - self.printSUCCESS(_("Nothing to merge")) + emerge.question.action = lambda x: False + emerge.run() + if not emerge.install_packages.list: + #self.printSUCCESS(_("Nothing to merge")) return True - self.printPre(update_list) + if ask_emerge: + self.printPre(str(emerge.install_packages)) + except EmergeError: + self.printPre(self._emerge_translate(emerge.prepare_error)) + raise + if (ask_emerge and self.askConfirm( + _("Would you like to merge these packages?")) == 'no'): + raise KeyboardInterrupt + emerge.command.send("yes\n") + + emerge.emerging.add_observer(self._printEmergePackage) + emerge.installing.add_observer(self._printInstallPackage) + emerge.uninstalling.add_observer(self._printUninstallPackage) + try: + emerge.run() except EmergeError as e: - if e.errno == EmergeError.CONFLICT: - self.printPre(emerge.update_block) - self.printPre(emerge.conflict_block) - elif e.errno == EmergeError.CUSTOM: - self.printPre(emerge.custom_error) + self.printPre(self._emerge_translate(emerge.emerging_error.log)) + raise + return True + + def depclean(self): + """ + Выполнить очистку системы от лишних пакетов + """ + with EmergeParser(EmergeCommand(["--depclean"])) as emerge: + try: + emerge.question.action = lambda x: False + emerge.run() + if not emerge.uninstall_packages.list: + return True + self._display_remove_list(emerge) + except EmergeError: + self.printPre(self._emerge_translate(emerge.prepare_error)) raise if (self.askConfirm( - _("Would you like to merge these packages?")) == 'no'): + _("Would you like to unmerge these packages?")) == 'no'): return False + emerge.command.send("yes\n") - - emerge.handle_emerging(self._printEmergePackage) - emerge.handle_installing(self._printInstallPackage) + emerge.uninstalling.add_observer(self._printUninstallPackage) try: - return emerge.install() + emerge.run() except EmergeError: - self.printPre( - convert_console_to_xml(readFile(emerge.get_error_log()))) + self.printPre(self._emerge_translate(emerge.emerging_error.log)) raise + return True + + def update_task(self, task_name): + """ + Декоратор для добавления меток запуска и останова задачи + """ + + def decor(f): + def wrapper(self, *args, **kwargs): + logger = EmergeLog(EmergeLogNamedTask(task_name)) + logger.mark_begin_task() + ret = f(self, *args, **kwargs) + if ret: + logger.mark_end_task() + return ret + + return wrapper + + return decor + diff --git a/update/utils/cl_update.py b/update/utils/cl_update.py index cb59e53..92a1ffe 100644 --- a/update/utils/cl_update.py +++ b/update/utils/cl_update.py @@ -17,9 +17,12 @@ import sys from calculate.core.server.func import Action, Tasks from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate +from calculate.lib.utils.colortext import get_color_print, Colors from calculate.lib.utils.files import FilesError -from calculate.update.update import UpdateError, EmergeError -from calculate.update.package_tools import GitError, Emerge +from calculate.update.update import UpdateError +from calculate.update.emerge_parser import EmergeError +from calculate.update.package_tools import GitError, Eix, EmergeLog, \ + EmergeLogNamedTask, PackageList setLocalTranslate('cl_update3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) @@ -36,81 +39,238 @@ class ClUpdateAction(Action): failedMessage = __("Update failed") interruptMessage = __("Update manually interrupted") + def was_installed(pkg, task_name): + def func(): + task = EmergeLog(EmergeLogNamedTask(task_name)) + return bool(PackageList(task.list)[pkg]) + return func + + def need_upgrade(pkg): + def func(): + return bool(Eix(pkg, Eix.Option.Upgrade).get_packages()) + return func + + def pkg_color(text): + _print = get_color_print() + return _print.bold.foreground(Colors.BLUE)(text) + + log_names = {'premerge': "check updates", + 'python_updater': "update python modules", + 'perl_cleaner': "update perl modules", + 'kernel_modules': "update kernel modules", + 'xorg_modules': "update xorg modules", + 'preserved_libs': "update preserved libs", + 'revdep': "revdep rebuild"} + emerge_tasks = [ - {'name': 'update_portage', - 'message': 'Update portage', - 'method': 'Update.emerge("-u","portage")', - 'condition': lambda : Emerge('-uOp','portage').has_update('sys-apps/portage') + {'name': 'premerge_group', + 'group': __("Checking updates"), + 'tasks': [ + {'name': 'premerge', + 'message': __("Calculating dependencies"), + 'method': 'Update.premerge("-uDN","--with-bdeps=y","@world")', + }], }, - {'name': 'update_python', - 'message': 'Update python', - 'method': 'Update.emerge("-u","python")', - 'condition': lambda : Emerge('-uOp','python').has_update('sys-apps/portage') >= "3.0" + {'name': 'update', + 'depend': (Tasks.result("premerge", eq='yes') | + Tasks.hasnot("premerge")) }, - - {'name': 'update_world', - 'message': __("Emerge world"), - 'method': 'Update.emerge("-uDN","@world")', - } - ] - - # список задач для дейсвия - tasks = [ - {'name': 'sync_reps', - 'foreach': 'cl_update_sync_rep', - 'message': __("Syncing {eachvar} repository"), - 'method': 'Update.syncRepositories(eachvar)', - 'condition': lambda Get: Get('cl_update_sync_rep') + {'name': 'update:update_portage', + 'group': __("Updating portage"), + 'tasks': [ + {'name': 'update:update_portage_pkg', + 'message': __("Updating {0}").format( + pkg_color("sys-apps/portage")), + 'method': 'Update.emerge("-u","portage")', + 'condition': need_upgrade('sys-apps/portage$') + }, + ] }, - {'name': 'sync_other_reps', - 'foreach': 'cl_update_other_rep_name', - 'message': __("Syncing {eachvar} repository"), - 'method': 'Update.syncLaymanRepository(eachvar)', - 'condition': lambda Get: Get('cl_update_other_set') == 'on' + {'name': 'update:update_python', + 'group': __("Updating python"), + 'tasks': [ + {'name': 'update:update_python_pkg', + 'message': __('Updating {0}').format( + pkg_color('dev-lang/python')), + 'method': 'Update.emerge("-u","dev-lang/python")', + 'condition': need_upgrade('dev-lang/python$') + }, + {'name': 'update:python_updater', + 'message': __('Updating python modules'), + 'method': 'Update.emergelike("python-updater")', + 'condition': was_installed('dev-lang/python$', + log_names['python_updater']), + 'decoration': 'Update.update_task("%s")' % log_names[ + 'python_updater'] + }, + ] }, - {'name': 'sync_reps:regen_cache', - 'foreach': 'cl_update_sync_overlay_rep', - 'message': __("Updating cache {eachvar} repository"), - 'essential': False, - 'method': 'Update.regenCache(eachvar)', - 'condition': lambda Get: (Get('cl_update_outdate_set') == 'on' and - Get('cl_update_metadata_force') != 'skip' or - Get('cl_update_metadata_force') == 'force') + {'name': 'update:update_perl', + 'group': __("Updating perl"), + 'tasks': [ + {'name': 'update:update_perl_pkg', + 'message': __('Updating {0}').format(pkg_color('dev-lang/perl')), + 'method': 'Update.emerge("-u","dev-lang/perl")', + 'condition': need_upgrade('dev-lang/perl$') + }, + {'name': 'update:perl_cleaner', + 'message': __('Updating perl modules'), + 'method': 'Update.emergelike("perl-cleaner", "all")', + 'condition': was_installed('dev-lang/perl$', + log_names['perl_cleaner']), + 'decoration': 'Update.update_task("%s")' % log_names[ + 'perl_cleaner'] + }, + ] }, - {'name': 'sync_other_reps:regen_other_cache', - 'foreach': 'cl_update_other_rep_name', - 'message': __("Updating cache {eachvar} repository"), - 'method': 'Update.regenCache(eachvar)', - 'essential': False, + {'name': 'update:update_calculate', + 'group': __("Updating Calculate Utilities"), + 'tasks': [ + {'name': 'update:update_calculate_pkgs', + 'message': __("Updating {0}").format( + pkg_color("sys-apps/calculate-utilities")), + 'method': 'Update.emerge("-u","sys-apps/calculate-utilities")', + 'condition': need_upgrade('sys-apps/calculate-utilities$') + }, + ] }, - {'name': 'emerge_metadata', - 'message': __("Metadata trasfering"), - 'method': 'Update.emergeMetadata()', - 'condition': lambda Get: (Get('cl_update_outdate_set') == 'on' and - Get('cl_update_metadata_force') != 'skip' or - Get('cl_update_metadata_force') == 'force') + {'name': 'update:update_world', + 'group': __("Updating all packages"), + 'tasks': [ + {'name': 'update:update_world', + 'message': __("Updating {0}").format(pkg_color("world")), + 'method': 'Update.emerge("-uDN","--with-bdeps=y","@world")', + }, + ] }, - {'name': 'eix_update', - 'message': __("Updating eix cache"), - 'method': 'Update.eixUpdate()', - 'condition': lambda Get: (Get('cl_update_outdate_set') == 'on' and - Get('cl_update_eixupdate_force') != 'skip' or - Get('cl_update_eixupdate_force') == 'force') + {'name': 'update:update_world', + 'group': __("Cleaning system from needless packages"), + 'tasks': [ + {'name': 'update:update_depclean', + 'message': __("Emerge depclean"), + 'method': 'Update.depclean()', + }, + ] }, - {'name': 'dispatch', - 'method': 'Update.applyTemplates(install.cl_source,' - 'cl_template_clt_set,True,None)', - 'condition': lambda Get: (Get('cl_update_rev_set') == 'on' or - Get('cl_rebuild_world_set') == 'on') + {'name': 'update:update_modules', + 'group': __("Rebuilding dependent modules"), + 'tasks': [ + {'name': 'update:module_rebuild', + 'message': __('Updating kernel modules'), + 'method': 'Update.emerge("@module-rebuild")', + 'condition': was_installed('sys-kernel/.*source', + log_names['kernel_modules']), + 'decoration': 'Update.update_task("%s")' % log_names[ + 'kernel_modules'] + }, + {'name': 'update:x11_module_rebuild', + 'message': __('Updating xorg-server modules'), + 'method': 'Update.emerge("@x11-module-rebuild")', + 'condition': was_installed('x11-base/xorg-server', + log_names['xorg_modules']), + 'decoration': 'Update.update_task("%s")' % log_names[ + 'xorg_modules'] + }, + {'name': 'update:preserved_rebuild', + 'message': __('Updating preserved libraries'), + 'method': 'Update.emerge("@preserved-rebuild")', + 'condition': was_installed('.*', log_names['preserved_libs']), + 'decoration': 'Update.update_task("%s")' % log_names[ + 'preserved_libs'] + }, + {'name': 'update:revdev_rebuild', + 'message': __('Executing {0}').format("revdep-rebuild"), + 'method': 'Update.emergelike("revdep-rebuild")', + 'condition': was_installed('.*', log_names['revdep']), + 'decoration': 'Update.update_task("%s")' % log_names['revdep'] + } + ] }, - # сообщение удачного завершения при обновлении репозиториев - {'name': 'success_syncrep', - 'message': __("Synchronization finished!"), - 'depend': (Tasks.success() & Tasks.has_any("sync_reps", - "sync_other_reps", - "emerge_metadata", - "eix_update")), - }] + emerge_tasks + [ + ] + + # список задач для дейсвия + tasks = [ + {'name': 'reps_synchronization', + 'group': __("Repository synchronization"), + 'tasks': [ + {'name': 'sync_reps', + 'foreach': 'cl_update_sync_rep', + 'message': __("Syncing {0} repository").format( + pkg_color("{{eachvar}}")), + 'method': 'Update.syncRepositories(eachvar)', + 'condition': lambda Get: Get('cl_update_sync_rep') + }, + {'name': 'sync_other_reps', + 'foreach': 'cl_update_other_rep_name', + 'message': __("Syncing {0} repository").format( + pkg_color("{{eachvar}}")), + 'method': 'Update.syncLaymanRepository(eachvar)', + 'condition': lambda Get: Get('cl_update_other_set') == 'on' + }, + {'name': 'sync_reps:regen_cache', + 'foreach': 'cl_update_sync_overlay_rep', + 'message': __("Updating cache {0} repository").format( + pkg_color("{{eachvar}}")), + 'essential': False, + 'method': 'Update.regenCache(eachvar)', + 'condition': ( + lambda Get: (Get('cl_update_outdate_set') == 'on' and + Get('cl_update_metadata_force') != 'skip' or + Get('cl_update_metadata_force') == 'force')) + }, + {'name': 'sync_other_reps:regen_other_cache', + 'foreach': 'cl_update_other_rep_name', + 'message': __("Updating cache {0} repository").format( + pkg_color("{{eachvar}}")), + 'method': 'Update.regenCache(eachvar)', + 'essential': False, + }, + {'name': 'emerge_metadata', + 'message': __("Metadata trasfering"), + 'method': 'Update.emergeMetadata()', + 'condition': ( + lambda Get: (Get('cl_update_outdate_set') == 'on' and + Get('cl_update_metadata_force') != 'skip' or + Get('cl_update_metadata_force') == 'force')) + }, + {'name': 'eix_update', + 'message': __("Updating eix cache"), + 'method': 'Update.eixUpdate()', + 'condition': ( + lambda Get: (Get('cl_update_outdate_set') == 'on' and + Get('cl_update_eixupdate_force') != 'skip' or + Get('cl_update_eixupdate_force') == 'force')) + }, + # сообщение удачного завершения при обновлении репозиториев + {'name': 'success_syncrep', + 'message': __("Synchronization finished"), + 'depend': (Tasks.success() & Tasks.has_any("sync_reps", + "sync_other_reps", + "emerge_metadata", + "eix_update")), + } + ] + }, + {'name': 'reps_synchronization', + 'group': __("System configuration"), + 'tasks': [ + {'name': 'revision', + 'message': __("Fixing settings"), + 'method': 'Update.applyTemplates(install.cl_source,' + 'cl_template_clt_set,True,None)', + 'condition': lambda Get: (Get('cl_update_rev_set') == 'on' or + Get('cl_rebuild_world_set') == 'on') + }, + {'name': 'world', + 'message': __("Update system packages list"), + 'method': 'Update.applyTemplates(install.cl_source,' + 'cl_template_clt_set,True,None)', + 'condition': lambda Get: (Get('cl_update_rev_set') == 'on' or + Get('cl_rebuild_world_set') == 'on') + }, + ] + } + ] + emerge_tasks + [ # сообщение удачного завершения при обновлении ревизии {'name': 'success_rev', 'message': __("System update finished!"), @@ -120,4 +280,5 @@ class ClUpdateAction(Action): {'name': 'success_world', 'message': __("World rebuild finished!"), 'condition': lambda Get: Get('cl_rebuild_world_set') == 'on' - }] + } + ] diff --git a/update/variables/update.py b/update/variables/update.py index a200b6a..abf7c97 100644 --- a/update/variables/update.py +++ b/update/variables/update.py @@ -24,6 +24,8 @@ from calculate.lib.utils.portage import searchProfile from calculate.lib.utils.files import readLinesFile, readFile from calculate.lib.cl_lang import setLocalTranslate +from calculate.update.emerge_parser import EmergeCache + setLocalTranslate('cl_update3',sys.modules[__name__]) class VariableAcUpdateSync(ReadonlyVariable): @@ -210,6 +212,7 @@ class VariableClUpdateBranchRep(ReadonlyVariable): def get(self): return self.Get('cl_update_rep_name') + class VariableClUpdateBranchName(Variable): """ Список доступных репозиторием @@ -220,15 +223,17 @@ class VariableClUpdateBranchName(Variable): self.label = _("Branch") def choice(self): - return ["master","develop","update"] + return ["master", "develop", "update"] def get(self): def generateBranch(): for reppath in self.Get('cl_update_rep_path'): - headPath = path.join(reppath,".git/HEAD") + headPath = path.join(reppath, ".git/HEAD") yield readFile(headPath).rpartition('/')[2].strip() or "master" + return list(generateBranch()) + class VariableClUpdateSyncRep(Variable): """ Обновляемый репозиторий @@ -380,3 +385,15 @@ class VariableClUpdateLaymanMake(Variable): """ # TODO: извлечь из layman.cfg value = "/var/lib/layman/make.conf" + +class VariableClUpdatePrecheckSet(Variable): + """ + Запустить предварительную проверку на обновления + """ + type = "bool" + value = "off" + opt = ["--pre-check"] + + def init(self): + self.help = _("Run pre-check updates") + self.label = _("run pre-check updates") diff --git a/update/wsdl_update.py b/update/wsdl_update.py index 5d9c169..27262c4 100644 --- a/update/wsdl_update.py +++ b/update/wsdl_update.py @@ -39,7 +39,7 @@ class Wsdl(WsdlBase): # категория метода 'category': __('Update'), # заголовок метода - 'title': __("Update system"), + 'title': __("Update System"), # иконка для графической консоли 'image': 'software-properties,preferences-desktop', # метод присутствует в графической консоли @@ -67,6 +67,7 @@ class Wsdl(WsdlBase): 'cl_update_sync_rep', 'cl_update_branch', 'cl_update_metadata_force', 'cl_update_other_set', + 'cl_update_precheck_set', 'cl_update_eixupdate_force', 'cl_templates_locate', 'cl_verbose_set', 'cl_dispatch_conf'),