# -*- coding: utf-8 -*- # Copyright 2014-2016 Mir Calculate. http://www.calculate-linux.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from functools import wraps import random import sys from os import path import os import time from calculate.core.server.gen_pid import search_worked_process from calculate.core.setup_cache import Cache as SetupCache from calculate.core.server.func import MethodsInterface from calculate.lib.cl_template import SystemIni from calculate.lib.datavars import DataVarsError from calculate.lib.utils.tools import AddonError from calculate.lib.utils.colortext.palette import TextState from calculate.lib.utils.colortext import get_color_print from calculate.update.emerge_parser import RevdepPercentBlock from calculate.update.datavars import DataVarsUpdate from calculate.update.update_info import UpdateInfo from calculate.lib.cl_log import log import re import shutil from collections import MutableSet from update_tasks import EmergeMark from calculate.lib.utils.git import Git, GitError, MTimeKeeper from calculate.lib.utils.portage import (Layman, EmergeLog, EmergeLogNamedTask, PackageList, PackageInformation, get_packages_files_directory, get_manifest_files_directory, get_remove_list) Colors = TextState.Colors from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir, PercentProgress, process, getRunCommands, readFile, listDirectory) import emerge_parser import logging from emerge_parser import (EmergeParser, EmergeCommand, EmergeError, EmergeCache, Chroot) from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, RegexpLocalization, _) setLocalTranslate('cl_update3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) class UpdateError(AddonError): """Update Error""" class OverlayOwnCache(MutableSet): """ Сет оверлеев с интегрированным кэшем """ def __init__(self, dv=None): self.dv = dv def __get_overlays(self): own_cache_value = SystemIni(self.dv).getVar('update', 'own_cache') or "" return [x.strip() for x in own_cache_value.split(',') if x.strip()] def __write_overlays(self, overlays): if not overlays: SystemIni(self.dv).delVar('update', 'own_cache') else: SystemIni(self.dv).setVar('update', {'own_cache': ",".join(overlays)}) def __contains__(self, item): return item in self.__get_overlays() def __iter__(self): return iter(self.__get_overlays()) def __len__(self): return len(self.__get_overlays()) def __append_value(self, overlays, value): if value not in overlays: overlays.append(value) self.__write_overlays(overlays) def add(self, value): overlays = self.__get_overlays() self.__append_value(overlays, value) def discard(self, value): overlays = self.__get_overlays() if value in overlays: overlays.remove(value) self.__write_overlays(overlays) def variable_module(var_env): def variable_module_decor(f): @wraps(f) def wrapper(self, *args, **kw): old_env = self.clVars.defaultModule try: self.clVars.defaultModule = var_env return f(self, *args, **kw) finally: self.clVars.defaultModule = old_env return wrapper return variable_module_decor class Update(MethodsInterface): """Основной объект для выполнения действий связанных с обновлением системы """ def init(self): commandLog = path.join(self.clVars.Get('core.cl_log_path'), 'lastcommand.log') emerge_parser.CommandExecutor.logfile = commandLog self.color_print = get_color_print() self.emerge_cache = EmergeCache() if self.clVars.Get('cl_env_debug_set') == 'off': EmergeCache.logger.logger.setLevel(logging.WARNING) self.emerge_cache.check_list = ( self.emerge_cache.check_list + map(emerge_parser.GitCheckvalue, self.clVars.Get('update.cl_update_rep_path'))) self.update_map = {} self.refresh_binhost = False def get_prog_path(self, program_name): return getProgPath(program_name) def _syncRepository(self, name, url, rpath, revision, cb_progress=None, clean=False): """ Синхронизировать репозитори """ dv = self.clVars git = Git() info_outdated = False old_dir = "%s.old" % git._gitDir(rpath) if path.exists(old_dir): clean = True try: self.stash_cache(rpath, name) if not git.checkExistsRep(rpath): git.cloneTagRepository(url, rpath, revision, cb_progress=cb_progress) info_outdated = True else: cr = "" try: need_update = False tag_cr = git.getCommit(rpath, revision) cr = git.getCurrentCommit(rpath) ref_type = git.reference_type(rpath, revision) if git.isNeedUnpack(rpath): need_update = True clean = True elif tag_cr != cr or ref_type == Git.Reference.Branch: need_update = True elif clean: status = git.getStatusInfo(rpath) if not status or status['files']: need_update = True except GitError: need_update = True if need_update: git.updateTagRepository(url, rpath, revision, cb_progress=cb_progress, clean=clean) new_cr = git.getCurrentCommit(rpath) if new_cr != cr: info_outdated = True if info_outdated: self.raiseOutdate() dv.Set('cl_update_outdate_set', 'on', force=True) finally: self.unstash_cache(rpath, name) return True def raiseOutdate(self): self.clVars.Set('cl_update_outdate_set', 'on', force=True) def setAutocheckParams(self, status, interval, update_other, cleanpkg): """ Настроить параметры автопроверки обновлений """ onoff = lambda x: "on" if x else "off" self.clVars.Write('cl_update_autocheck_set', onoff(status), True) self.clVars.Write('cl_update_autocheck_interval', interval, True) self.clVars.Write('cl_update_other_set', onoff(update_other), True) self.clVars.Write('cl_update_cleanpkg_set', onoff(cleanpkg), True) if not status: UpdateInfo.set_update_ready(False) return True def checkSchedule(self, interval, status): """ Проверить по расписанию необходимость запуска команды """ if not status: self.printWARNING(_("Updates autocheck is not enabled")) return False last_check = SystemIni().getVar('update', 'last_check') or "" re_interval = re.compile("^(\d+)\s*(hours?|days?|weeks?)?", re.I) interval_match = re_interval.search(interval) MINUTE = 60 HOUR = MINUTE * 60 DAY = HOUR * 24 WEEK = DAY * 7 if interval_match: if interval_match.group(2): suffix_map = {'h': HOUR, 'd': DAY, 'w': WEEK} k = suffix_map.get(interval_match.group(2).lower()[0], HOUR) else: k = HOUR est = int(interval_match.group(1)) * k else: est = 3 * HOUR if last_check: if last_check.isdigit(): if (time.time() - int(last_check)) < (est - 10 * MINUTE): self.printWARNING(_("Please wait for the update time")) return False self.mark_schedule() return True def checkRun(self, wait_update): """ Проверить повторный запуск """ update_running = lambda: any(os.getpid() != x for x in search_worked_process('update', dv)) dv = self.clVars if update_running(): if not wait_update: raise UpdateError(_("Update is already running. " "Try to run later.")) else: self.startTask(_("Waiting for another update to be complete")) while update_running(): self.pauseProcess() while update_running(): time.sleep(0.3) self.resumeProcess() time.sleep(random.random() * 3) self.endTask() if self.clVars.Get('cl_chroot_status') == 'off': emerge_running = lambda: any("/usr/bin/emerge" in x for x in getRunCommands(True)) if emerge_running(): if not wait_update: raise UpdateError(_("Emerge is running. " "Try to run later.")) else: self.startTask(_("Waiting for emerge to be complete")) while emerge_running(): time.sleep(1) self.endTask() return True @variable_module("update") def trimRepositories(self, repname): """ Синхронизировать репозитории """ dv = self.clVars rpath = \ dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1) git = Git() self.addProgress() git.trimRepository(rpath, cb_progress=self.setProgress) return True @variable_module("update") def syncRepositories(self, repname, clean_on_error=True): """ Синхронизировать репозитории """ dv = self.clVars check_status = dv.GetBool('update.cl_update_check_rep_set') url, rpath, revision = ( dv.Select(["cl_update_rep_url", "cl_update_rep_path", "cl_update_rep_rev"], where="cl_update_rep_name", eq=repname, limit=1)) if not url or not rpath: raise UpdateError(_("Configuration variables for repositories " "are not setup")) git = Git() if not git.checkUrl(url): raise UpdateError(_("Git %s is unavailable") % url) chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) if chroot_path == '/': rpath_orig = rpath else: rpath_orig = rpath[len(chroot_path):] self.addProgress() mtime = MTimeKeeper(path.join(rpath, "profiles/updates")) mtime.save() try: if clean_on_error: try: layman = Layman(dv.Get('cl_update_layman_installed'), dv.Get('cl_update_layman_make'), dv.Get('cl_update_layman_conf'), prefix=chroot_path) if repname != "portage": layman.add(repname, url, rpath_orig) if not self._syncRepository(repname, url, rpath, revision, cb_progress=self.setProgress, clean=check_status): return "skip" return True except GitError as e: if e.addon: self.printWARNING(str(e.addon)) self.printWARNING(str(e)) self.endTask(False) self.startTask( _("Re-fetching the {name} repository").format( name=repname)) self.addProgress() rpath_new = "%s_new" % rpath try: self._syncRepository(repname, url, rpath_new, revision, cb_progress=self.setProgress, clean=check_status) removeDir(rpath) shutil.move(rpath_new, rpath) except OSError as e: raise UpdateError(_("Failed to modify the " "{repname} repository").format( repname=repname) + _(": ") + str(e)) finally: if path.exists(rpath_new): removeDir(rpath_new) else: if not self._syncRepository(repname, url, rpath, revision, clean=check_status): return "skip" layman = Layman(dv.Get('cl_update_layman_installed'), dv.Get('cl_update_layman_make'), dv.Get('cl_update_layman_conf'), prefix=chroot_path) if repname != "portage": layman.add(repname, url, rpath_orig) finally: mtime.restore() return True metadata_cache_names = ("metadata/md5-cache", "metadata/cache") def stash_cache(self, rpath, name): """ Спрятать кэш """ if name in ("portage",): return if not name in OverlayOwnCache(self.clVars): for cachename in self.metadata_cache_names: cachedir = path.join(rpath, cachename) if path.exists(cachedir): try: cachedir_s = path.join(path.dirname(rpath), path.basename( cachename) + ".stash") if path.exists(cachedir_s): removeDir(cachedir_s) shutil.move(cachedir, cachedir_s) except BaseException as e: pass def unstash_cache(self, rpath, name): """ Извлеч кэш """ if name in ("portage",): return cachenames = self.metadata_cache_names if not name in OverlayOwnCache(self.clVars): if any(path.exists(path.join(rpath, x)) for x in cachenames): for cachename in cachenames: cachedir_s = path.join(path.dirname(rpath), path.basename(cachename) + ".stash") if path.exists(cachedir_s): try: removeDir(cachedir_s) except BaseException as e: pass OverlayOwnCache(self.clVars).add(name) else: for cachename in cachenames: cachedir = path.join(rpath, cachename) cachedir_s = path.join(path.dirname(rpath), path.basename(cachename) + ".stash") if path.exists(cachedir_s): try: shutil.move(cachedir_s, cachedir) except BaseException as e: pass else: if all(not path.exists(path.join(rpath, x)) for x in cachenames): OverlayOwnCache(self.clVars).discard(name) def syncLaymanRepository(self, repname): """ Обновить репозиторий через layman """ layman = self.get_prog_path('/usr/bin/layman') if not layman: raise UpdateError(_("The Layman tool is not found")) rpath = self.clVars.Select('cl_update_other_rep_path', where='cl_update_other_rep_name', eq=repname, limit=1) laymanname = path.basename(rpath) self.stash_cache(rpath, laymanname) try: if Git.is_git(rpath): self.addProgress() p = PercentProgress(layman, "-s", laymanname, part=1, atty=True) for perc in p.progress(): self.setProgress(perc) else: p = process(layman, "-s", repname, stderr=STDOUT) if p.failed(): raise UpdateError( _("Failed to update the {rname} repository").format( rname=repname), addon=p.read()) finally: self.unstash_cache(rpath, laymanname) return True def _regenCache_process(self, progname, repname, cpu_num): return process(progname, "--repo=%s" % repname, "--update", "--jobs=%s" % cpu_num, stderr=STDOUT) def regenCache(self, repname): """ Обновить кэш метаданных репозитория """ egenCache = self.get_prog_path('/usr/bin/egencache') if not egenCache: raise UpdateError(_("The Portage tool is not found")) if repname in self.clVars.Get('cl_update_rep_name'): path_rep = self.clVars.Select('cl_update_rep_path', where='cl_update_rep_name', eq=repname, limit=1) repo_name = readFile( path.join(path_rep, "profiles/repo_name")).strip() if repo_name != repname: self.printWARNING( _("Repository '{repo_name}' called '{repname}'" " in cl_update_rep_name").format( repo_name=repo_name, repname=repname)) raise UpdateError(_("Failed to update the cache of the {rname} " "repository").format(rname=repname)) cpu_num = self.clVars.Get('hr_cpu_num') if repname in OverlayOwnCache(self.clVars): self.printWARNING( _("Repository %s has its own cache") % repname.capitalize()) else: self.startTask(_("Updating the %s repository cache") % repname.capitalize()) p = self._regenCache_process(egenCache, repname, cpu_num) if p.failed(): raise UpdateError(_("Failed to update the cache of the {rname} " "repository").format(rname=repname), addon=p.read()) return True def emergeMetadata(self): """ Выполнить egencache и emerge --metadata """ emerge = self.get_prog_path("/usr/bin/emerge") if not emerge: raise UpdateError(_("The Emerge tool is not found")) self.addProgress() p = PercentProgress(emerge, "--ask=n", "--metadata", part=1, atty=True) for perc in p.progress(): self.setProgress(perc) if p.failed(): data = p.read() with open('/var/log/calculate/failed-metadata-%d.log' % time.time(), 'w') as f: f.write(data + p.alldata) raise UpdateError(_("Failed to update metadata"), addon=data) return True def _eixUpdateCommand(self, eix_cmd, countRep): return PercentProgress(eix_cmd, "-F", part=countRep or 1, atty=True) def eixUpdate(self, repositroies): """ Выполенине eix-update для репозиторием eix-update выполнятется только для тех репозиториев, которые обновлялись, если cl_update_eixsync_force==auto, либо все, если cl_update_eixupdate_force==force """ eixupdate = self.get_prog_path("/usr/bin/eix-update") if not eixupdate: raise UpdateError(_("The Eix tool is not found")) self.addProgress() countRep = len(repositroies) p = self._eixUpdateCommand(eixupdate, countRep) for perc in p.progress(): self.setProgress(perc) if p.failed(): raise UpdateError(_("Failed to update eix cache"), addon=p.read()) return True def is_binary_pkg(self, pkg, binary=None): """ Является ли пакет бинарным """ if binary: return True if 'PN' in pkg and pkg['PN'].endswith('-bin'): return True if binary is not None: return binary if "binary" in pkg and pkg['binary']: return True return False def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1): """ Вывод сообщения сборки пакета """ self.endTask() _print = self.color_print one = _print("{0}", num) two = _print("{0}", max_num) part = _("({current} of {maximum})").format(current=one, maximum=two) _print = _print.foreground(Colors.DEFAULT) if self.is_binary_pkg(pkg, binary): _colorprint = _print.foreground(Colors.PURPLE) else: _colorprint = _print.foreground(Colors.GREEN) PackageInformation.add_info(pkg) name = "" if pkg.info['DESCRIPTION']: name = _(pkg.info['DESCRIPTION']) name = name[:1].upper() + name[1:] if not name: name = str(pkg) self.printSUCCESS( _("{part} {package}").format(part=part, package=name)) self.startTask( _("Emerging {package}").format(package=_colorprint(str(pkg)))) def _printInstallPackage(self, pkg, binary=False): """ Вывод сообщения установки пакета """ self.endTask() _print = self.color_print if self.is_binary_pkg(pkg, binary): _print = _print.foreground(Colors.PURPLE) else: _print = _print.foreground(Colors.GREEN) pkg_key = "{CATEGORY}/{PF}".format(**pkg) if pkg_key in self.update_map: self.startTask(_("Installing {pkg} [{oldver}]").format( pkg=_print(str(pkg)), oldver=self.update_map[pkg_key])) self.update_map.pop(pkg_key) else: self.startTask(_("Installing %s") % (_print(str(pkg)))) def _printFetching(self, fn): """ Вывод сообщения о скачивании """ self.endTask() self.startTask(_("Fetching binary packages")) def _printUninstallPackage(self, pkg, num=1, max_num=1): """ Вывод сообщения удаления пакета """ self.endTask() _print = self.color_print if num and max_num: one = _print("{0}", num) two = _print("{0}", max_num) part = _(" ({current} of {maximum})").format(current=one, maximum=two) else: part = "" _print = _print.foreground(Colors.LIGHT_RED) self.startTask( _("Unmerging{part} {package}").format(part=part, package=_print(str(pkg)))) def emergelike(self, cmd, *params): """ Запуск команды, которая подразумевает выполнение emerge """ cmd_path = self.get_prog_path(cmd) if not cmd_path: raise UpdateError(_("Failed to find the %s command") % cmd) with EmergeParser( emerge_parser.CommandExecutor(cmd_path, params)) as emerge: self._startEmerging(emerge) return True def revdep_rebuild(self, cmd, *params): """ Запуск revdep-rebulid """ cmd_path = self.get_prog_path(cmd) if not cmd_path: raise UpdateError(_("Failed to find the %s command") % cmd) with EmergeParser( emerge_parser.CommandExecutor(cmd_path, params)) as emerge: revdep = RevdepPercentBlock(emerge) self.addProgress() revdep.add_observer(self.setProgress) revdep.action = lambda x: ( self.endTask(), self.startTask(_("Assigning files to packages")) if "Assign" in revdep else None) self._startEmerging(emerge) return True def _display_pretty_package_list(self, pkglist, remove_list=False): """ Отобразить список пакетов в "удобочитаемом" виде """ _print = self.color_print ebuild_color = TextState.Colors.GREEN binary_color = TextState.Colors.PURPLE remove_color = TextState.Colors.LIGHT_RED flag_map = {"updating": _print.foreground(TextState.Colors.LIGHT_CYAN)("U"), "reinstall": _print.foreground(TextState.Colors.YELLOW)("rR"), "new": _print.foreground(TextState.Colors.LIGHT_GREEN)("N"), "newslot": _print.foreground(TextState.Colors.LIGHT_GREEN)("NS"), "downgrading": ( _print.foreground(TextState.Colors.LIGHT_CYAN)("U") + _print.foreground(TextState.Colors.LIGHT_BLUE)("D"))} for pkg in sorted([PackageInformation.add_info(x) for x in pkglist], key=lambda y: y['CATEGORY/PN']): install_flag = "" if remove_list: pkgcolor = _print.foreground(remove_color) else: for flag in flag_map: if pkg[flag]: install_flag = "(%s) " % flag_map[flag] break if self.is_binary_pkg(pkg): pkgcolor = _print.foreground(binary_color) else: pkgcolor = _print.foreground(ebuild_color) if pkg.info['DESCRIPTION']: fullname = "%s " % _(pkg.info['DESCRIPTION']) fullname = fullname[:1].upper() + fullname[1:] else: fullname = "" shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"])) if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB": size = " (%s)" % pkg["SIZE"] else: size = "" mult = _print.bold("*") self.printDefault( " {mult} {fullname}{flag}{shortname}{size}".format( mult=mult, fullname=fullname, shortname=shortname, size=size, flag=install_flag)) def _display_install_package(self, emerge, emergelike=False): """ Отобразить список устанавливаемых пакетов """ # подробный список пакетов _print = self.color_print if emergelike: self.printPre(str(emerge.install_packages)) else: pkglist = emerge.install_packages.list self.printSUCCESS(_print( _("Listing packages for installation"))) self._display_pretty_package_list(pkglist) if emerge.install_packages.remove_list: self.printSUCCESS(_print( _("Listing packages for removal"))) self._display_pretty_package_list( emerge.install_packages.remove_list, remove_list=True) if len(emerge.install_packages.list) > 0: install_mess = (_("{count} packages will be installed").format( count=len(emerge.install_packages.list)) + ", ") else: install_mess = "" if str(emerge.download_size) != "0 kB": self.printSUCCESS(_("{install}{size} will be downloaded").format( install=install_mess, size=str(emerge.download_size))) def _display_remove_list(self, emerge): """ Отобразить список удаляемых пакетов """ # подробный список пакетов if self.clVars.Get('update.cl_update_emergelist_set') == 'on': self.printPre(self._emerge_translate( emerge.uninstall_packages.verbose_result)) else: _print = self.color_print pkglist = emerge.uninstall_packages.list self.printSUCCESS(_print.bold( _("Listing packages for removal"))) self._display_pretty_package_list(pkglist, remove_list=True) def getCacheOnWorld(self, params, packages, check=False): """ Получить список обновляемых пакетов @world из кэша """ if "@world" in packages: elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge)) if check and (elog.list or elog.remove_list): self.emerge_cache.drop_cache( "Some packages was installed or removed") return params, packages installed_pkgs = elog.list new_packages = self.emerge_cache.get_cached_package_list() if new_packages is not None: return "-1O", ["=%s" % x for x in new_packages if not str(x) in installed_pkgs] return params, packages def updateCache(self, pkg_list): """ Обновить кэш. Оставить отметку в emerge.log о том, выполнено действие premerge """ self.emerge_cache.set_cache(pkg_list) elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge)) elog.mark_end_task(), def mark_schedule(self): """ Установить отметку о запуске запланированной проверки """ SystemIni().setVar('update', {'last_check': str(int(time.time()))}) def get_default_emerge_opts(self): return self.clVars.Get('cl_emerge_default_opts') def premerge(self, param, *packages): """ Вывести информацию об обновлении """ deo = self.clVars.Get('cl_emerge_default_opts') param, packages = self.getCacheOnWorld(param, packages, check=True) param = [param, "-pv"] if not packages: self.printSUCCESS(_("Installed packages are up to date")) self.set_need_update(False) return True with EmergeParser(EmergeCommand(list(packages), emerge_default_opts=deo, extra_params=param)) as emerge: try: emerge.run() if "@world" in packages: if emerge.install_packages.remove_list: self.emerge_cache.drop_cache( "List has packages for remove") elif emerge.install_packages.block_packages: self.emerge_cache.drop_cache( "List has block packages") else: self.updateCache(emerge.install_packages.list) if not emerge.install_packages.list: self.printSUCCESS(_("The system is up to date")) self.set_need_update(False) return True emergelike = self.clVars.Get('cl_update_emergelist_set') == 'on' self._display_install_package(emerge, emergelike) if str(emerge.skipped_packages): self._display_error(emerge.skipped_packages) except EmergeError: self.set_need_update(False) self.emerge_cache.drop_cache("Emerge error") self._display_install_package(emerge, emergelike=True) self._display_error(emerge.prepare_error) raise if self.clVars.Get('cl_update_pretend_set') == 'on': # установить кэш: есть обновления self.set_need_update() return True self.set_need_update(False) answer = self.askConfirm( _("Would you like to merge these packages?"), "yes") if answer == "no": raise KeyboardInterrupt return "yes" return True def set_need_update(self, val=True): """ Установить флаг: есть обновления """ if self.clVars.Get('update.cl_update_autocheck_set') == 'off': val = False UpdateInfo.set_update_ready(val) return True def _emerge_translate(self, s): """ Перевести текст из emerge """ return RegexpLocalization('cl_emerge').translate(str(s)) def setUpToDateCache(self): """ Установить кэш - "нет пакетов для обновления" """ self.updateCache(PackageList([])) return True def _startEmerging(self, emerge): """ Настроить и выполнить emerge """ if emerge.install_packages and emerge.install_packages.list: for pkg in emerge.install_packages.list: rv = pkg.get('REPLACING_VERSIONS', '') if rv: self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \ rv.partition(":")[0] emerge.command.send("yes\n") emerge.emerging.add_observer(self._printEmergePackage) emerge.installing.add_observer(self._printInstallPackage) emerge.uninstalling.add_observer(self._printUninstallPackage) emerge.fetching.add_observer(self._printFetching) def cancel_observing_fetch(fn): emerge.fetching.clear_observers() emerge.fetching.add_observer(cancel_observing_fetch) try: emerge.run() except EmergeError: self.emerge_cache.drop_cache("Emerge error") if emerge.emerging_error: self._display_error(emerge.emerging_error.log) else: self._display_error(emerge.prepare_error) raise def _display_error(self, error): lines_num = int(self.clVars.Get('update.cl_update_lines_limit')) error = "
".join(str(error).split('
')[-lines_num:]) self.printPre(self._emerge_translate(error)) def emerge(self, use, param, *packages): """ Выполнить сборку пакета """ deo = self.clVars.Get('cl_emerge_default_opts') if not packages: packages = [param] extra_params = None else: param, packages = self.getCacheOnWorld(param, packages) if not packages: return True extra_params = [param] command = EmergeCommand(list(packages), emerge_default_opts=deo, extra_params=extra_params, use=use) if self.clVars.Get('cl_chroot_path') != '/': command = Chroot(self.clVars.Get('cl_chroot_path'), command) with EmergeParser(command) as emerge: try: emerge.question.action = lambda x: False emerge.run() if not emerge.install_packages.list: return True except EmergeError: self.emerge_cache.drop_cache("Emerge error") self._display_error(emerge.prepare_error) raise self._startEmerging(emerge) return True def depclean(self): """ Выполнить очистку системы от лишних пакетов """ deo = self.get_default_emerge_opts() emerge = None try: emerge = EmergeParser(EmergeCommand(["--depclean"], emerge_default_opts=deo)) outdated_kernel = False try: emerge.question.action = lambda x: False emerge.run() if not emerge.uninstall_packages.list: UpdateInfo(self.clVars).outdated_kernel = False return True kernel_pkg = self.clVars.Get('cl_update_kernel_pkg') if any(("%s-%s" % (x['CATEGORY/PN'], x['PVR'])) == kernel_pkg for x in emerge.uninstall_packages.list): pkglist = [ "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in emerge.uninstall_packages.list if ("%s-%s" % (x['CATEGORY/PN'], x['PVR'])) != kernel_pkg] emerge.command.send('n\n') emerge.close() emerge = None if not pkglist: UpdateInfo(self.clVars).outdated_kernel = True return True emerge = EmergeParser( EmergeCommand(pkglist, extra_params=["--unmerge", '--ask=y'], emerge_default_opts=deo)) emerge.question.action = lambda x: False emerge.run() outdated_kernel = True else: outdated_kernel = False self._display_remove_list(emerge) except EmergeError: self._display_error(emerge.prepare_error) raise if (self.askConfirm( _("Would you like to unmerge these unused packages " "(recommended)?")) != 'yes'): return True UpdateInfo(self.clVars).outdated_kernel = outdated_kernel self._startEmerging(emerge) finally: if emerge: emerge.close() return True def update_task(self, task_name): """ Декоратор для добавления меток запуска и останова задачи """ def decor(f): def wrapper(*args, **kwargs): logger = EmergeLog(EmergeLogNamedTask(task_name)) logger.mark_begin_task() ret = f(*args, **kwargs) if ret: logger.mark_end_task() return ret return wrapper return decor def migrateCacheRepository(self, url, branch, storage): """ Перенести репозиторий из кэша в локальный """ rep = storage.get_repository(url, branch) if rep: rep.storage = storage.storages[0] self.clVars.Invalidate('cl_update_profile_storage') return True def reconfigureProfileVars(self, profile_dv, chroot): """ Синхронизировать репозитории """ dv = self.clVars try: if not profile_dv: raise UpdateError( _("Failed to use the new profile. Try again.")) for var_name in ('cl_update_rep_path', 'cl_update_rep_url', 'cl_update_rep_name', 'cl_update_branch', 'cl_update_binhost_list', 'cl_update_branch_name', 'cl_profile_system', 'cl_update_rep'): dv.Set(var_name, profile_dv.Get(var_name), force=True) dv.Set('cl_chroot_path', chroot, force=True) except DataVarsError as e: print str(e) raise UpdateError(_("Wrong profile")) return True def setProfile(self, profile_shortname): profile = self.clVars.Select('cl_update_profile_path', where='cl_update_profile_shortname', eq=profile_shortname, limit=1) if not profile: raise UpdateError(_("Failed to determine profile %s") % self.clVars.Get('cl_update_profile_system')) profile_path = path.relpath(profile, '/etc/portage') try: profile_file = '/etc/portage/make.profile' if not path.exists( path.join(path.dirname(profile_file), profile_path)): raise UpdateError( _("Failed to set the profile: %s") % _("Profile not found")) for rm_fn in filter(path.lexists, ('/etc/make.profile', '/etc/portage/make.profile')): os.unlink(rm_fn) os.symlink(profile_path, profile_file) except (OSError, IOError) as e: raise UpdateError(_("Failed to set the profile: %s") % str(e)) return True def applyProfileTemplates(self, useClt=None, cltFilter=False, useDispatch=True, action="merge"): """ Наложить шаблоны из профиля """ from calculate.lib.cl_template import TemplatesError, ProgressTemplate dv = DataVarsUpdate() try: dv.importUpdate() dv.flIniFile() dv.Set('cl_action', action, force=True) dv.Set('cl_templates_locate', self.clVars.Get('cl_update_templates_locate')) dv.Set("cl_chroot_path", '/', True) dv.Set("cl_root_path", '/', True) for copyvar in ("cl_dispatch_conf", "cl_verbose_set", "update.cl_update_world"): dv.Set(copyvar, self.clVars.Get(copyvar), True) # определение каталогов содержащих шаблоны useClt = useClt in ("on", True) self.addProgress() nullProgress = lambda *args, **kw: None dispatch = self.dispatchConf if useDispatch else None clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt, cltFilter=cltFilter, printSUCCESS=self.printSUCCESS, printWARNING=self.printWARNING, askConfirm=self.askConfirm, dispatchConf=dispatch, printERROR=self.printERROR) try: clTempl.applyTemplates() if clTempl.hasError(): if clTempl.getError(): raise TemplatesError(clTempl.getError()) finally: if clTempl: if clTempl.cltObj: clTempl.cltObj.closeFiles() clTempl.closeFiles() finally: dv.close() return True def cleanpkg(self): """ Очистить PKGDIR и DISTFILES в текущей системе """ portdirs = ([self.clVars.Get('cl_portdir')] + self.clVars.Get('cl_portdir_overlay')) pkgfiles = get_packages_files_directory(*portdirs) distdirfiles = get_manifest_files_directory(*portdirs) distdir = self.clVars.Get('install.cl_distfiles_path') pkgdir = self.clVars.Get('cl_pkgdir') logger = log("update_cleanpkg.log", filename="/var/log/calculate/update_cleanpkg.log", formatter="%(asctime)s - %(clean)s - %(message)s") return self._cleanpkg( distdir, pkgdir, distdirfiles, pkgfiles, logger) def _update_binhost_packages(self): os.system('/usr/sbin/emaint binhost -f &>/dev/null') def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger): """ Общий алгоритм очистки distfiles и pkgdir от устаревших пакетов """ skip_files = ["/metadata.dtd", "/Packages"] try: if self.clVars.Get('client.os_remote_auth'): skip_files += ['portage_lockfile'] except DataVarsError: pass for cleantype, filelist in ( ("packages", get_remove_list(pkgdir, list(pkgfiles), depth=4)), ("distfiles", get_remove_list(distdir, list(distdirfiles), depth=1))): removelist = [] for fn in filelist: try: if not any(fn.endswith(x) for x in skip_files): os.unlink(fn) removelist.append(path.basename(fn)) except OSError: pass removelist_str = ",".join(removelist) if removelist_str: logger.info(removelist_str, extra={'clean': cleantype}) if cleantype == "packages": try: self._update_binhost_packages() for dn in listDirectory(pkgdir, fullPath=True): if path.isdir(dn) and not listDirectory(dn): os.rmdir(dn) except OSError: pass return True def updateSetupCache(self): cache = SetupCache(self.clVars) cache.update(force=True) return True def check_binhost(self, write_binhost=True): """ Проверить, что доступен хотя бы один из binhost'ов :return: """ hosts = self.clVars.Get("update.cl_update_binhost_host") if not hosts: self.clVars.Delete('cl_update_binhost', location="system") raise UpdateError(_("Update server is unavailable")) if write_binhost: if hosts[0] != self.clVars.Get('update.cl_update_binhost'): self.refresh_binhost = True self.clVars.Write('cl_update_binhost', hosts[0], location="system") new_ts = self.clVars.Get("update.cl_update_binhost_timestamp") if new_ts: new_ts = new_ts[0] old_ts = self.clVars.Get("update.cl_update_last_timestamp") if new_ts.isdigit() and (not old_ts.isdigit() or int(new_ts) > int(old_ts)): ini = SystemIni(self.clVars) ini.setVar('update', {'last_update': new_ts}) return True def message_binhost_changed(self): if self.refresh_binhost: self.printWARNING(_("Update server was changed to %s") % self.clVars.Get('cl_update_binhost')) elif self.clVars.GetBool('update.cl_update_binhost_recheck_set'): self.printSUCCESS(_("Update server %s") % self.clVars.Get('cl_update_binhost')) return True def update_binhost_list(self): """ Обновить список binhost'ов после обновления до master веток :return: """ dv = DataVarsUpdate() try: dv.importUpdate() dv.flIniFile() changes = False for varname in ('update.cl_update_binhost_list', 'update.cl_update_binhost_timestamp_path', 'cl_update_binhost_revision_path'): new_value = dv.Get(varname) old_value = self.clVars.Get(varname) if new_value != old_value: changes = True self.clVars.Set(varname, new_value, force=True) if not changes: raise UpdateError(_("Update server is unavailable")) except DataVarsError: raise UpdateError(_("Update server is unavailable")) return True def drop_binhosts(self, dv): """ Обновление до master веток """ branch = dv.Get('update.cl_update_branch') revs = [ branch for x in dv.Get('update.cl_update_rep_name') ] dv.Set('update.cl_update_branch_name', revs) dv.Invalidate('update.cl_update_rep_rev') return True