Non puoi selezionare più di 25 argomenti Gli argomenti devono iniziare con una lettera o un numero, possono includere trattini ('-') e possono essere lunghi fino a 35 caratteri.
calculate-utils-3-update/update/update.py

732 righe
29 KiB

Questo file contiene caratteri Unicode ambigui!

Questo file contiene caratteri Unicode ambigui che possono essere confusi con altri nella tua localizzazione attuale. Se il tuo caso di utilizzo è intenzionale e legittimo, puoi tranquillamente ignorare questo avviso. Usa il pulsante Escape per evidenziare questi caratteri.

#-*- coding: utf-8 -*-
# Copyright 2014 Calculate Ltd. http://www.calculate-linux.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from itertools import ifilter
import random
import sys
from os import path
import os
import time
from calculate.core.server.gen_pid import search_worked_process, ProcessStatus
from calculate.lib.cl_template import SystemIni
from calculate.lib.datavars import DataVarsError
from calculate.lib.utils.tools import AddonError
from calculate.lib.utils.colortext.palette import TextState
from calculate.lib.utils.colortext import get_color_print
from calculate.update.emerge_parser import RevdepPercentBlock
import math
from package_tools import Git, Layman,\
EmergeLogNamedTask, EmergeLog, GitError, \
PackageInformation, PackageList, EmergePackage
Colors = TextState.Colors
from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
PercentProgress, process, getRunCommands,
readFile, listDirectory)
import emerge_parser
import logging
from emerge_parser import EmergeParser, EmergeCommand, EmergeError, EmergeCache
from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
RegexpLocalization, _)
setLocalTranslate('cl_update3', sys.modules[__name__])
__ = getLazyLocalTranslate(_)
class UpdateError(AddonError):
"""Update Error"""
class Update:
"""Основной объект для выполнения действий связанных с обновлением системы
"""
def init(self):
commandLog = path.join(self.clVars.Get('core.cl_log_path'),
'lastcommand.log')
emerge_parser.CommandExecutor.logfile = commandLog
self.color_print = get_color_print()
self.emerge_cache = EmergeCache()
if self.clVars.Get('cl_env_debug_set') == 'off':
EmergeCache.logger.logger.setLevel(logging.WARNING)
self.emerge_cache.check_list = (
self.emerge_cache.check_list +
map(emerge_parser.GitCheckvalue,
self.clVars.Get('update.cl_update_rep_path')))
self.update_map = {}
def _syncRepository(self, name, url, rpath, revision, branch,
cb_progress=None):
"""
Синхронизировать репозитори
"""
dv = self.clVars
git = Git()
needMeta = False
if not git.checkExistsRep(rpath):
if revision == "last":
git.cloneRepository(url, rpath, branch,
cb_progress=cb_progress)
else:
git.cloneRevRepository(url, rpath, branch, revision,
cb_progress=cb_progress)
needMeta = True
else:
# если нужно обновиться до конкретной ревизии
if revision != "last":
if revision == git.getCurrentCommit(rpath):
if git.getBranch(rpath) == branch:
return False
# получить изменения из удаленного репозитория
git.fetchRepository(rpath, cb_progress=cb_progress)
# если текущая ветка не соответствует нужной
repInfo = git.getStatusInfo(rpath)
if repInfo['branch'] != branch:
# меняем ветку
needMeta = True
git.checkoutBranch(rpath, branch)
if revision == "last":
if git.resetRepository(rpath, to_origin=True):
# если не удалось сбросить
repInfo = git.getStatusInfo(rpath)
if repInfo.get("files", False):
raise GitError("Failed to reset git")
needMeta = True
else:
git.resetRepository(rpath, to_rev=revision)
needMeta = True
if needMeta:
dv.Set('cl_update_outdate_set', 'on', force=True)
return True
def checkRun(self, wait_update):
"""
Проверить повторный запуск
"""
update_running = lambda: any(os.getpid() != x
for x in search_worked_process('update', dv))
dv = self.clVars
if update_running():
if not wait_update:
raise UpdateError(_("Update is already running. "
"Try to run later."))
else:
self.startTask(_("Waiting for another update to be complete"))
while update_running():
self.pauseProcess()
while update_running():
time.sleep(0.3)
self.resumeProcess()
time.sleep(random.random()*3)
self.endTask()
emerge_running = lambda: any("/usr/bin/emerge" in x
for x in getRunCommands())
if emerge_running():
if not wait_update:
raise UpdateError(_("Emerge is running. "
"Try to run later."))
else:
self.startTask(_("Waiting for emerge to be complete"))
while emerge_running():
time.sleep(1)
self.endTask()
return True
def syncRepositories(self, repname, clean_on_error=True):
"""
Синхронизировать репозитории
"""
dv = self.clVars
url, rpath, revision, branch = (
dv.Select(["cl_update_rep_url", "cl_update_rep_path",
"cl_update_rep_rev", "cl_update_branch_name"],
where="cl_update_rep_name", eq=repname, limit=1))
if not url or not rpath:
raise UpdateError(_("Configuration variables for repositories "
"are not setup"))
self.addProgress()
if clean_on_error:
try:
if not self._syncRepository(repname, url, rpath, revision, branch,
cb_progress=self.setProgress):
return "skip"
layman = Layman(dv.Get('cl_update_layman_installed'),
dv.Get('cl_update_layman_make'))
if repname != "portage":
layman.add(repname, url, rpath)
return True
except GitError as e:
if e.addon:
self.printWARNING(str(e.addon))
self.printWARNING(str(e))
self.endTask(False)
self.startTask(
_("Re-fetching the {name} repository").format(name=repname))
self.addProgress()
try:
rpath_new = "%s_new" % rpath
self._syncRepository(repname, url, rpath_new, revision,
branch, cb_progress=self.setProgress)
removeDir(rpath)
os.rename(rpath_new, rpath)
except OSError as e:
raise UpdateError(_("Failed to modify the "
"{repname} repository").format(
repname=repname)+":"+str(e))
finally:
if path.exists(rpath_new):
removeDir(rpath_new)
else:
if not self._syncRepository(repname, url, rpath, revision, branch):
return "skip"
layman = Layman(dv.Get('cl_update_layman_installed'),
dv.Get('cl_update_layman_make'))
if repname != "portage":
layman.add(repname, url, rpath)
return True
def syncLaymanRepository(self, repname):
"""
Обновить репозиторий через layman
"""
layman = getProgPath('/usr/bin/layman')
if not layman:
raise UpdateError(_("The Layman tool is not found"))
rpath = self.clVars.Select('cl_update_other_rep_path',
where='cl_update_other_rep_name', eq=repname,
limit=1)
laymanname = path.basename(rpath)
if Git.is_git(rpath):
self.addProgress()
p = PercentProgress(layman, "-s", laymanname, part=1, atty=True)
for perc in p.progress():
self.setProgress(perc)
else:
p = process(layman, "-s", repname, stderr=STDOUT)
if p.failed():
raise UpdateError(
_("Failed to update the {rname} repository").format(rname=repname),
addon=p.read())
return True
def regenCache(self, repname):
"""
Обновить кэш метаданных репозитория
"""
egenCache = getProgPath('/usr/bin/egencache')
if not egenCache:
raise UpdateError(_("The Portage tool is not found"))
if repname in self.clVars.Get('cl_update_rep_name'):
path_rep = self.clVars.Select('cl_update_rep_path',
where='cl_update_rep_name',
eq=repname, limit=1)
repo_name = readFile(
path.join(path_rep,"profiles/repo_name")).strip()
if repo_name != repname:
self.printWARNING(
_("Repository '{repo_name}' called '{repname}'"
" in cl_update_rep_name").format(
repo_name=repo_name, repname=repname))
raise UpdateError(_("Failed to update the cache of the {rname} "
"repository").format(rname=repname))
cpu_num = self.clVars.Get('hr_cpu_num')
p = process(egenCache, "--repo=%s" % repname, "--update",
"--jobs=%s" % cpu_num, stderr=STDOUT)
if p.failed():
raise UpdateError(_("Failed to update the cache of the {rname} "
"repository").format(rname=repname),
addon=p.read())
return True
def emergeMetadata(self):
"""
Выполнить egencache и emerge --metadata
"""
emerge = getProgPath("/usr/bin/emerge")
if not emerge:
raise UpdateError(_("The Emerge tool is not found"))
self.addProgress()
p = PercentProgress(emerge, "--ask=n", "--metadata", part=1, atty=True)
for perc in p.progress():
self.setProgress(perc)
if p.failed():
data = p.read()
with open('/var/log/calculate/failed-metadata-%d.log' % time.time(),
'w') as f:
f.write(data+p.alldata)
raise UpdateError(_("Failed to update metadata"), addon=data)
return True
def eixUpdate(self):
"""
Выполенине eix-update для репозиторием
eix-update выполнятется только для тех репозиториев, которые
обновлялись, если cl_update_eixsync_force==auto, либо
все, если cl_update_eixupdate_force==force
"""
eixupdate = getProgPath("/usr/bin/eix-update")
if not eixupdate:
raise UpdateError(_("The Eix tool is not found"))
self.addProgress()
countRep = len(self.clVars.Get('main.cl_portdir_overlay'))+1
p = PercentProgress(eixupdate, "-F", part=countRep or 1, atty=True)
for perc in p.progress():
self.setProgress(perc)
if p.failed():
raise UpdateError(_("Failed to update eix cache"), addon=p.read())
return True
def is_binary_pkg(self, pkg, binary=None):
"""
Является ли пакет бинарным
"""
if binary:
return True
if 'PN' in pkg and pkg['PN'].endswith('-bin'):
return True
if binary is not None:
return binary
if "binary" in pkg and pkg['binary']:
return True
return False
def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
"""
Вывод сообщения сборки пакета
"""
self.endTask()
_print = self.color_print
one = _print("{0}", num)
two = _print("{0}", max_num)
part = _("({current} of {maximum})").format(current=one,
maximum=two)
_print = _print.foreground(Colors.DEFAULT)
if self.is_binary_pkg(pkg,binary):
_colorprint = _print.foreground(Colors.PURPLE)
else:
_colorprint = _print.foreground(Colors.GREEN)
PackageInformation.add_info(pkg)
name = ""
if pkg.info['DESCRIPTION']:
name = _(pkg.info['DESCRIPTION'])
name = name[:1].upper() + name[1:]
if not name:
name = str(pkg)
self.printSUCCESS(
_("{part} {package}").format(part=part, package=name))
self.startTask(
_("Emerging {package}").format(package=_colorprint(str(pkg))))
def _printInstallPackage(self, pkg, binary=False):
"""
Вывод сообщения установки пакета
"""
self.endTask()
_print = self.color_print
if self.is_binary_pkg(pkg,binary):
_print = _print.foreground(Colors.PURPLE)
else:
_print = _print.foreground(Colors.GREEN)
#print listDirectory('/var/db/pkg/%s' % pkg['CATEGORY'])
pkg_key = "{CATEGORY}/{PF}".format(**pkg)
if pkg_key in self.update_map:
self.startTask(_("Installing {pkg} [{oldver}]").format(
pkg=_print(str(pkg)), oldver=self.update_map[ pkg_key]))
else:
self.startTask(_("Installing %s") % (_print(str(pkg))))
def _printFetching(self, fn):
"""
Вывод сообщения о скачивании
"""
self.endTask()
self.startTask(_("Fetching binary packages"))
def _printUninstallPackage(self, pkg, num=1, max_num=1):
"""
Вывод сообщения удаления пакета
"""
self.endTask()
_print = self.color_print
one = _print("{0}", num)
two = _print("{0}", max_num)
part = _(" ({current} of {maximum})").format(current=one,
maximum=two)
_print = _print.foreground(Colors.LIGHT_RED)
self.startTask(
_("Unmerging{part} {package}").format(part=part,
package=_print(str(pkg))))
def emergelike(self, cmd, *params):
"""
Запуск команды, которая подразумевает выполнение emerge
"""
cmd_path = getProgPath(cmd)
if not cmd_path:
raise UpdateError(_("Failed to find the %s command") % cmd)
with EmergeParser(
emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
self._startEmerging(emerge)
return True
def revdep_rebuild(self, cmd, *params):
"""
Запуск revdep-rebulid
"""
cmd_path = getProgPath(cmd)
if not cmd_path:
raise UpdateError(_("Failed to find the %s command") % cmd)
with EmergeParser(
emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
revdep = RevdepPercentBlock(emerge)
self.addProgress()
revdep.add_observer(self.setProgress)
revdep.action = lambda x: (
self.endTask(), self.startTask(_("Assigning files to packages"))
if "Assign" in revdep else None)
self._startEmerging(emerge)
return True
def _display_pretty_package_list(self, pkglist, remove_list=False):
"""
Отобразить список пакетов в "удобочитаемом" виде
"""
_print = self.color_print
ebuild_color = TextState.Colors.GREEN
binary_color = TextState.Colors.PURPLE
remove_color = TextState.Colors.LIGHT_RED
flag_map = {"updating":
_print.foreground(TextState.Colors.LIGHT_CYAN)("U"),
"reinstall":
_print.foreground(TextState.Colors.YELLOW)("rR"),
"new":
_print.foreground(TextState.Colors.LIGHT_GREEN)("N"),
"newslot":
_print.foreground(TextState.Colors.LIGHT_GREEN)("NS"),
"downgrading": (
_print.foreground(TextState.Colors.LIGHT_CYAN)("U") +
_print.foreground(TextState.Colors.LIGHT_BLUE)("D"))}
for pkg in sorted([PackageInformation.add_info(x) for x in
pkglist],
key=lambda y: y['CATEGORY/PN']):
install_flag = ""
if remove_list:
pkgcolor = _print.foreground(remove_color)
else:
for flag in flag_map:
if pkg[flag]:
install_flag = "(%s) " % flag_map[flag]
break
if self.is_binary_pkg(pkg):
pkgcolor = _print.foreground(binary_color)
else:
pkgcolor = _print.foreground(ebuild_color)
if pkg.info['DESCRIPTION']:
fullname = "%s " % _(pkg.info['DESCRIPTION'])
fullname = fullname[:1].upper()+fullname[1:]
else:
fullname = ""
shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB":
size = " (%s)" % pkg["SIZE"]
else:
size = ""
mult = _print.bold("*")
self.printDefault(
" {mult} {fullname}{flag}{shortname}{size}".format(
mult=mult, fullname=fullname, shortname=shortname, size=size,
flag=install_flag))
def _display_install_package(self, emerge):
"""
Отобразить список устанавливаемых пакетов
"""
# подробный список пакетов
_print = self.color_print
if self.clVars.Get('cl_verbose_set') == 'on':
self.printPre(str(emerge.install_packages))
else:
pkglist = emerge.install_packages.list
self.printSUCCESS(_print(
_("Listing packages for installation")))
self._display_pretty_package_list(pkglist)
if emerge.install_packages.remove_list:
self.printSUCCESS(_print(
_("Listing packages for removal")))
self._display_pretty_package_list(
emerge.install_packages.remove_list, remove_list=True)
if str(emerge.download_size) != "0 kB":
self.printSUCCESS(_("{size} will be downloaded").format(
size=str(emerge.download_size)))
def _display_remove_list(self, emerge):
"""
Отобразить список удаляемых пакетов
"""
# подробный список пакетов
if self.clVars.Get('cl_verbose_set') == 'on':
self.printPre(self._emerge_translate(
emerge.uninstall_packages.verbose_result))
else:
_print = self.color_print
pkglist = emerge.uninstall_packages.list
self.printSUCCESS(_print.bold(
_("Listing packages for removal")))
self._display_pretty_package_list(pkglist, remove_list=True)
def getCacheOnWorld(self, params, packages, check=False):
"""
Получить список обновляемых пакетов @world из кэша
"""
if "@world" in packages:
from calculate.update.utils.cl_update import ClUpdateAction
elog = EmergeLog(
EmergeLogNamedTask(ClUpdateAction.log_names['premerge']))
if check and (elog.list or elog.remove_list):
self.emerge_cache.drop_cache(
"Some packages was installed or removed")
return params, packages
installed_pkgs = elog.list
new_packages = self.emerge_cache.get_cached_package_list()
if new_packages is not None:
return "-1O", ["=%s" % x for x in new_packages
if not str(x) in installed_pkgs]
return params, packages
def updateCache(self, pkg_list):
"""
Обновить кэш. Оставить отметку в emerge.log о том, выполнено действие
premerge
"""
self.emerge_cache.set_cache(pkg_list)
from calculate.update.utils.cl_update import ClUpdateAction
elog = EmergeLog(
EmergeLogNamedTask(ClUpdateAction.log_names['premerge']))
elog.mark_end_task(),
def premerge(self, param, *packages):
"""
Вывести информацию об обновлении
"""
param, packages = self.getCacheOnWorld(param, packages, check=True)
param = [param, "-pv"]
if not packages:
self.printSUCCESS(_("Installed packages are up to date"))
self.set_need_update(False)
return True
with EmergeParser(EmergeCommand(list(packages),
extra_params=param)) as emerge:
try:
emerge.run()
if "@world" in packages:
if emerge.install_packages.remove_list:
self.emerge_cache.drop_cache(
"List has packages for remove")
else:
self.updateCache(emerge.install_packages.list)
if not emerge.install_packages.list:
self.printSUCCESS(_("The system is up to date"))
self.set_need_update(False)
return True
self._display_install_package(emerge)
except EmergeError:
self.set_need_update(False)
self.emerge_cache.drop_cache("Emerge error")
self.printPre(self._emerge_translate(emerge.prepare_error))
raise
if self.clVars.Get('cl_update_pretend_set') == 'on':
# установить кэш: есть обновления
self.set_need_update()
return True
self.set_need_update(False)
answer = self.askConfirm(
_("Would you like to merge these packages?"), "yes")
if answer == "no":
raise KeyboardInterrupt
return "yes"
return True
def set_need_update(self, val=True):
"""
Установить флаг: есть обновления
"""
val = "on" if val else "off"
SystemIni().setVar('update', {'packages': val})
return True
def _emerge_translate(self, s):
"""
Перевести текст из emerge
"""
return RegexpLocalization('cl_emerge').translate(str(s))
def setUpToDateCache(self):
"""
Установить кэш - "нет пакетов для обновления"
"""
self.updateCache(PackageList([]))
return True
def _startEmerging(self, emerge):
"""
Настроить и выполнить emerge
"""
if emerge.install_packages and emerge.install_packages.list:
for pkg in emerge.install_packages.list:
rv = pkg.get('REPLACING_VERSIONS', '')
if rv:
self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \
rv.partition(":")[0]
emerge.command.send("yes\n")
emerge.emerging.add_observer(self._printEmergePackage)
emerge.installing.add_observer(self._printInstallPackage)
emerge.uninstalling.add_observer(self._printUninstallPackage)
emerge.fetching.add_observer(self._printFetching)
try:
emerge.run()
except EmergeError:
self.emerge_cache.drop_cache("Emerge error")
if emerge.emerging_error:
self.printPre(
self._emerge_translate(emerge.emerging_error.log))
else:
self.printPre(self._emerge_translate(emerge.prepare_error))
raise
def emerge(self, param, *packages):
"""
Выполнить сборку пакета
"""
if not packages:
packages = [param]
extra_params = None
else:
param, packages = self.getCacheOnWorld(param, packages)
if not packages:
return True
extra_params = [param]
with EmergeParser(EmergeCommand(list(packages),
extra_params=extra_params)) as emerge:
try:
emerge.question.action = lambda x: False
emerge.run()
if not emerge.install_packages.list:
return True
except EmergeError:
self.emerge_cache.drop_cache("Emerge error")
self.printPre(self._emerge_translate(emerge.prepare_error))
raise
self._startEmerging(emerge)
return True
def depclean(self):
"""
Выполнить очистку системы от лишних пакетов
"""
with EmergeParser(EmergeCommand(["--depclean"])) as emerge:
try:
emerge.question.action = lambda x: False
emerge.run()
if not emerge.uninstall_packages.list:
return True
self._display_remove_list(emerge)
except EmergeError:
self.printPre(self._emerge_translate(emerge.prepare_error))
raise
if (self.askConfirm(
_("Would you like to unmerge these packages?")) != 'yes'):
return False
self._startEmerging(emerge)
return True
def update_task(self, task_name):
"""
Декоратор для добавления меток запуска и останова задачи
"""
def decor(f):
def wrapper(*args, **kwargs):
logger = EmergeLog(EmergeLogNamedTask(task_name))
logger.mark_begin_task()
ret = f(*args, **kwargs)
if ret:
logger.mark_end_task()
return ret
return wrapper
return decor
def migrateCacheRepository(self, url, branch):
rep_set = self.clVars.Get('cl_update_profile_storage')
rep = rep_set.get_repository(url, branch)
if rep:
rep.storage = rep_set.storages[0]
self.clVars.Invalidate('cl_update_profile_storage')
return True
def reconfigureProfileVars(self):
"""
Синхронизировать репозитории
"""
dv = self.clVars
profile_dv = dv.Get('cl_update_profile_datavars')
try:
for var_name in ('cl_update_rep_rev',
'cl_update_rep_path',
'cl_update_rep_url',
'cl_update_rep_name',
'cl_update_branch_name',
'cl_profile_system',
'cl_update_layman_storage',
'cl_update_rep'):
dv.Set(var_name, profile_dv.Get(var_name), force=True)
except DataVarsError:
raise UpdateError(_("Wrong profile"))
return True
def setProfile(self):
profile = self.clVars.Select('cl_profile_path',
where='cl_profile_shortname',
eq=self.clVars.Get('cl_update_profile_system'), limit=1)
if not profile:
raise UpdateError(_("Failed to determine profile %s") %
self.clVars.Get('cl_update_profile_system'))
profile_path = path.relpath(profile, '/etc/portage')
try:
for rm_fn in filter(path.exists,
('/etc/make.profile', '/etc/portage/make.profile')):
os.unlink(rm_fn)
os.symlink(profile_path, '/etc/portage/make.profile')
except (OSError,IOError) as e:
raise UpdateError(_("Failed to set profile: %s")%str(e))
return True