|
|
@ -13,6 +13,7 @@ |
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
|
|
# See the License for the specific language governing permissions and |
|
|
|
# limitations under the License. |
|
|
|
from functools import wraps, partial |
|
|
|
import random |
|
|
|
|
|
|
|
import sys |
|
|
@ -32,8 +33,9 @@ from calculate.update.datavars import DataVarsUpdate |
|
|
|
from calculate.update.update_info import UpdateInfo |
|
|
|
from calculate.lib.cl_log import log |
|
|
|
import re |
|
|
|
from itertools import chain |
|
|
|
from collections import MutableSet |
|
|
|
from mock import self |
|
|
|
from update_tasks import EmergeMark |
|
|
|
|
|
|
|
from calculate.lib.utils.portage import (Git, Layman, EmergeLog, GitError, |
|
|
|
EmergeLogNamedTask, PackageList, |
|
|
@ -48,7 +50,8 @@ from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir, |
|
|
|
readFile, listDirectory) |
|
|
|
import emerge_parser |
|
|
|
import logging |
|
|
|
from emerge_parser import EmergeParser, EmergeCommand, EmergeError, EmergeCache |
|
|
|
from emerge_parser import (EmergeParser, EmergeCommand, EmergeError, |
|
|
|
EmergeCache, ChrootEmergeCommand) |
|
|
|
|
|
|
|
from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, |
|
|
|
RegexpLocalization, _) |
|
|
@ -63,18 +66,19 @@ class OverlayOwnCache(MutableSet): |
|
|
|
""" |
|
|
|
Сет оверлеев с интегрированным кэшем |
|
|
|
""" |
|
|
|
def __init__(self, initvalue=()): |
|
|
|
pass |
|
|
|
def __init__(self, dv=None): |
|
|
|
self.dv = dv |
|
|
|
|
|
|
|
def __get_overlays(self): |
|
|
|
own_cache_value = SystemIni().getVar('update', 'own_cache') or "" |
|
|
|
own_cache_value = SystemIni(self.dv).getVar('update', 'own_cache') or "" |
|
|
|
return [x.strip() for x in own_cache_value.split(',') if x.strip()] |
|
|
|
|
|
|
|
def __write_overlays(self, overlays): |
|
|
|
if not overlays: |
|
|
|
SystemIni().delVar('update', 'own_cache') |
|
|
|
SystemIni(self.dv).delVar('update', 'own_cache') |
|
|
|
else: |
|
|
|
SystemIni().setVar('update', {'own_cache': ",".join(overlays)}) |
|
|
|
SystemIni(self.dv).setVar('update', |
|
|
|
{'own_cache': ",".join(overlays)}) |
|
|
|
|
|
|
|
def __contains__(self, item): |
|
|
|
return item in self.__get_overlays() |
|
|
@ -101,6 +105,21 @@ class OverlayOwnCache(MutableSet): |
|
|
|
self.__write_overlays(overlays) |
|
|
|
|
|
|
|
|
|
|
|
def variable_module(var_env): |
|
|
|
def variable_module_decor(f): |
|
|
|
@wraps(f) |
|
|
|
def wrapper(self, *args, **kw): |
|
|
|
old_env = self.clVars.defaultModule |
|
|
|
try: |
|
|
|
self.clVars.defaultModule = var_env |
|
|
|
return f(self, *args, **kw) |
|
|
|
finally: |
|
|
|
self.clVars.defaultModule = old_env |
|
|
|
return wrapper |
|
|
|
|
|
|
|
return variable_module_decor |
|
|
|
|
|
|
|
|
|
|
|
class Update(object): |
|
|
|
"""Основной объект для выполнения действий связанных с обновлением системы |
|
|
|
|
|
|
@ -119,6 +138,9 @@ class Update(object): |
|
|
|
self.clVars.Get('update.cl_update_rep_path'))) |
|
|
|
self.update_map = {} |
|
|
|
|
|
|
|
def get_prog_path(self, progname): |
|
|
|
return getProgPath(progname) |
|
|
|
|
|
|
|
def _syncRepository(self, name, url, rpath, revision, branch, |
|
|
|
cb_progress=None): |
|
|
|
""" |
|
|
@ -131,36 +153,58 @@ class Update(object): |
|
|
|
self.stash_cache(rpath, name) |
|
|
|
if not git.checkExistsRep(rpath): |
|
|
|
if revision == "last": |
|
|
|
git.cloneRepository(url, rpath, branch, |
|
|
|
git.cloneTagRepository(url, rpath, branch, |
|
|
|
cb_progress=cb_progress) |
|
|
|
else: |
|
|
|
git.cloneRevRepository(url, rpath, branch, revision, |
|
|
|
git.cloneTagRepository(url, rpath, revision, |
|
|
|
cb_progress=cb_progress) |
|
|
|
info_outdate = True |
|
|
|
else: |
|
|
|
# если нужно обновиться до конкретной ревизии |
|
|
|
if revision != "last": |
|
|
|
if revision == git.getCurrentCommit(rpath): |
|
|
|
if git.getBranch(rpath) == branch: |
|
|
|
return False |
|
|
|
# получить изменения из удаленного репозитория |
|
|
|
git.fetchRepository(rpath, cb_progress=cb_progress) |
|
|
|
# если текущая ветка не соответствует нужной |
|
|
|
repInfo = git.getStatusInfo(rpath) |
|
|
|
if repInfo['branch'] != branch: |
|
|
|
# меняем ветку |
|
|
|
info_outdate = True |
|
|
|
git.checkoutBranch(rpath, branch) |
|
|
|
if revision == "last": |
|
|
|
if git.resetRepository(rpath, to_origin=True): |
|
|
|
# если не удалось сбросить |
|
|
|
repInfo = git.getStatusInfo(rpath) |
|
|
|
if repInfo.get("files", False): |
|
|
|
raise GitError("Failed to reset git") |
|
|
|
try: |
|
|
|
need_update = False |
|
|
|
tag_cr = git.getCommit(rpath, revision) |
|
|
|
cr = git.getCurrentCommit(rpath) |
|
|
|
if tag_cr != cr: |
|
|
|
need_update = True |
|
|
|
except GitError: |
|
|
|
need_update = True |
|
|
|
if need_update: |
|
|
|
git.updateTagRepository(url, rpath, revision, |
|
|
|
cb_progress=cb_progress) |
|
|
|
info_outdate = True |
|
|
|
else: |
|
|
|
git.resetRepository(rpath, to_rev=revision) |
|
|
|
info_outdate = True |
|
|
|
try: |
|
|
|
old_cr = git.getCurrentCommit(rpath) |
|
|
|
except GitError: |
|
|
|
old_cr = "" |
|
|
|
git.updateTagRepository(url, rpath, branch, |
|
|
|
cb_progress=cb_progress) |
|
|
|
if old_cr != git.getCurrentCommit(rpath): |
|
|
|
info_outdate = True |
|
|
|
# если нужно обновиться до конкретной ревизии |
|
|
|
#if revision != "last": |
|
|
|
# if revision == git.getCurrentCommit(rpath): |
|
|
|
# if git.getBranch(rpath) == branch: |
|
|
|
# return False |
|
|
|
## получить изменения из удаленного репозитория |
|
|
|
#git.fetchRepository(rpath, cb_progress=cb_progress) |
|
|
|
## если текущая ветка не соответствует нужной |
|
|
|
#repInfo = git.getStatusInfo(rpath) |
|
|
|
#if repInfo['branch'] != branch: |
|
|
|
# # меняем ветку |
|
|
|
# info_outdate = True |
|
|
|
# git.checkoutBranch(rpath, branch) |
|
|
|
#if revision == "last": |
|
|
|
# if git.resetRepository(rpath, to_origin=True): |
|
|
|
# # если не удалось сбросить |
|
|
|
# repInfo = git.getStatusInfo(rpath) |
|
|
|
# if repInfo.get("files", False): |
|
|
|
# raise GitError("Failed to reset git") |
|
|
|
# info_outdate = True |
|
|
|
#else: |
|
|
|
# git.resetRepository(rpath, to_rev=revision) |
|
|
|
# info_outdate = True |
|
|
|
if info_outdate: |
|
|
|
dv.Set('cl_update_outdate_set', 'on', force=True) |
|
|
|
finally: |
|
|
@ -247,6 +291,7 @@ class Update(object): |
|
|
|
self.endTask() |
|
|
|
return True |
|
|
|
|
|
|
|
@variable_module("update") |
|
|
|
def syncRepositories(self, repname, clean_on_error=True): |
|
|
|
""" |
|
|
|
Синхронизировать репозитории |
|
|
@ -259,6 +304,11 @@ class Update(object): |
|
|
|
if not url or not rpath: |
|
|
|
raise UpdateError(_("Configuration variables for repositories " |
|
|
|
"are not setup")) |
|
|
|
chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) |
|
|
|
if chroot_path == '/': |
|
|
|
rpath_orig = rpath |
|
|
|
else: |
|
|
|
rpath_orig = rpath[len(chroot_path):] |
|
|
|
self.addProgress() |
|
|
|
if clean_on_error: |
|
|
|
try: |
|
|
@ -266,7 +316,7 @@ class Update(object): |
|
|
|
dv.Get('cl_update_layman_make'), |
|
|
|
dv.Get('cl_update_layman_conf')) |
|
|
|
if repname != "portage": |
|
|
|
layman.add(repname, url, rpath) |
|
|
|
layman.add(repname, url, rpath_orig) |
|
|
|
if not self._syncRepository(repname, url, rpath, revision, branch, |
|
|
|
cb_progress=self.setProgress): |
|
|
|
return "skip" |
|
|
@ -279,8 +329,8 @@ class Update(object): |
|
|
|
self.startTask( |
|
|
|
_("Re-fetching the {name} repository").format(name=repname)) |
|
|
|
self.addProgress() |
|
|
|
rpath_new = "%s_new" % rpath |
|
|
|
try: |
|
|
|
rpath_new = "%s_new" % rpath |
|
|
|
self._syncRepository(repname, url, rpath_new, revision, |
|
|
|
branch, cb_progress=self.setProgress) |
|
|
|
removeDir(rpath) |
|
|
@ -300,7 +350,9 @@ class Update(object): |
|
|
|
dv.Get('cl_update_layman_make'), |
|
|
|
dv.Get('cl_update_layman_conf')) |
|
|
|
if repname != "portage": |
|
|
|
layman.add(repname, url, rpath) |
|
|
|
# TODO: debug block |
|
|
|
#print "EEEE",repname, url, rpath_orig |
|
|
|
layman.add(repname, url, rpath_orig) |
|
|
|
return True |
|
|
|
|
|
|
|
metadata_cache_names = ("metadata/md5-cache", "metadata/cache") |
|
|
@ -311,7 +363,7 @@ class Update(object): |
|
|
|
""" |
|
|
|
if name in ("portage",): |
|
|
|
return |
|
|
|
if not name in OverlayOwnCache(): |
|
|
|
if not name in OverlayOwnCache(self.clVars): |
|
|
|
for cachename in self.metadata_cache_names: |
|
|
|
cachedir = path.join(rpath, cachename) |
|
|
|
if path.exists(cachedir): |
|
|
@ -332,7 +384,7 @@ class Update(object): |
|
|
|
if name in ("portage",): |
|
|
|
return |
|
|
|
cachenames = self.metadata_cache_names |
|
|
|
if not name in OverlayOwnCache(): |
|
|
|
if not name in OverlayOwnCache(self.clVars): |
|
|
|
if any(path.exists(path.join(rpath, x)) for x in cachenames): |
|
|
|
for cachename in cachenames: |
|
|
|
cachedir_s = path.join(path.dirname(rpath), |
|
|
@ -342,7 +394,7 @@ class Update(object): |
|
|
|
removeDir(cachedir_s) |
|
|
|
except BaseException as e: |
|
|
|
pass |
|
|
|
OverlayOwnCache().add(name) |
|
|
|
OverlayOwnCache(self.clVars).add(name) |
|
|
|
else: |
|
|
|
for cachename in cachenames: |
|
|
|
cachedir = path.join(rpath, cachename) |
|
|
@ -355,14 +407,14 @@ class Update(object): |
|
|
|
pass |
|
|
|
else: |
|
|
|
if all(not path.exists(path.join(rpath, x)) for x in cachenames): |
|
|
|
OverlayOwnCache().discard(name) |
|
|
|
OverlayOwnCache(self.clVars).discard(name) |
|
|
|
|
|
|
|
|
|
|
|
def syncLaymanRepository(self, repname): |
|
|
|
""" |
|
|
|
Обновить репозиторий через layman |
|
|
|
""" |
|
|
|
layman = getProgPath('/usr/bin/layman') |
|
|
|
layman = self.get_prog_path('/usr/bin/layman') |
|
|
|
if not layman: |
|
|
|
raise UpdateError(_("The Layman tool is not found")) |
|
|
|
rpath = self.clVars.Select('cl_update_other_rep_path', |
|
|
@ -386,11 +438,15 @@ class Update(object): |
|
|
|
self.unstash_cache(rpath, laymanname) |
|
|
|
return True |
|
|
|
|
|
|
|
def _regenCache_process(self, progname, repname, cpu_num): |
|
|
|
return process(progname, "--repo=%s" % repname, "--update", |
|
|
|
"--jobs=%s" % cpu_num, stderr=STDOUT) |
|
|
|
|
|
|
|
def regenCache(self, repname): |
|
|
|
""" |
|
|
|
Обновить кэш метаданных репозитория |
|
|
|
""" |
|
|
|
egenCache = getProgPath('/usr/bin/egencache') |
|
|
|
egenCache = self.get_prog_path('/usr/bin/egencache') |
|
|
|
if not egenCache: |
|
|
|
raise UpdateError(_("The Portage tool is not found")) |
|
|
|
if repname in self.clVars.Get('cl_update_rep_name'): |
|
|
@ -407,14 +463,13 @@ class Update(object): |
|
|
|
raise UpdateError(_("Failed to update the cache of the {rname} " |
|
|
|
"repository").format(rname=repname)) |
|
|
|
cpu_num = self.clVars.Get('hr_cpu_num') |
|
|
|
if repname in OverlayOwnCache(): |
|
|
|
if repname in OverlayOwnCache(self.clVars): |
|
|
|
self.printWARNING( |
|
|
|
_("Repository %s has its own cache") % repname.capitalize()) |
|
|
|
else: |
|
|
|
self.startTask(_("Updating the %s repository cache") % |
|
|
|
repname.capitalize()) |
|
|
|
p = process(egenCache, "--repo=%s" % repname, "--update", |
|
|
|
"--jobs=%s" % cpu_num, stderr=STDOUT) |
|
|
|
p = self._regenCache_process(egenCache, repname, cpu_num) |
|
|
|
if p.failed(): |
|
|
|
raise UpdateError(_("Failed to update the cache of the {rname} " |
|
|
|
"repository").format(rname=repname), |
|
|
@ -425,7 +480,7 @@ class Update(object): |
|
|
|
""" |
|
|
|
Выполнить egencache и emerge --metadata |
|
|
|
""" |
|
|
|
emerge = getProgPath("/usr/bin/emerge") |
|
|
|
emerge = self.get_prog_path("/usr/bin/emerge") |
|
|
|
if not emerge: |
|
|
|
raise UpdateError(_("The Emerge tool is not found")) |
|
|
|
self.addProgress() |
|
|
@ -440,7 +495,10 @@ class Update(object): |
|
|
|
raise UpdateError(_("Failed to update metadata"), addon=data) |
|
|
|
return True |
|
|
|
|
|
|
|
def eixUpdate(self): |
|
|
|
def _eixUpdateCommand(self, eix_cmd, countRep): |
|
|
|
return PercentProgress(eix_cmd, "-F", part=countRep or 1, atty=True) |
|
|
|
|
|
|
|
def eixUpdate(self, repositroies): |
|
|
|
""" |
|
|
|
Выполенине eix-update для репозиторием |
|
|
|
|
|
|
@ -448,12 +506,12 @@ class Update(object): |
|
|
|
обновлялись, если cl_update_eixsync_force==auto, либо |
|
|
|
все, если cl_update_eixupdate_force==force |
|
|
|
""" |
|
|
|
eixupdate = getProgPath("/usr/bin/eix-update") |
|
|
|
eixupdate = self.get_prog_path("/usr/bin/eix-update") |
|
|
|
if not eixupdate: |
|
|
|
raise UpdateError(_("The Eix tool is not found")) |
|
|
|
self.addProgress() |
|
|
|
countRep = len(self.clVars.Get('main.cl_portdir_overlay'))+1 |
|
|
|
p = PercentProgress(eixupdate, "-F", part=countRep or 1, atty=True) |
|
|
|
countRep = len(repositroies) |
|
|
|
p = self._eixUpdateCommand(eixupdate, countRep) |
|
|
|
for perc in p.progress(): |
|
|
|
self.setProgress(perc) |
|
|
|
if p.failed(): |
|
|
@ -552,7 +610,7 @@ class Update(object): |
|
|
|
""" |
|
|
|
Запуск команды, которая подразумевает выполнение emerge |
|
|
|
""" |
|
|
|
cmd_path = getProgPath(cmd) |
|
|
|
cmd_path = self.get_prog_path(cmd) |
|
|
|
if not cmd_path: |
|
|
|
raise UpdateError(_("Failed to find the %s command") % cmd) |
|
|
|
with EmergeParser( |
|
|
@ -564,7 +622,7 @@ class Update(object): |
|
|
|
""" |
|
|
|
Запуск revdep-rebulid |
|
|
|
""" |
|
|
|
cmd_path = getProgPath(cmd) |
|
|
|
cmd_path = self.get_prog_path(cmd) |
|
|
|
if not cmd_path: |
|
|
|
raise UpdateError(_("Failed to find the %s command") % cmd) |
|
|
|
with EmergeParser( |
|
|
@ -662,7 +720,7 @@ class Update(object): |
|
|
|
Отобразить список удаляемых пакетов |
|
|
|
""" |
|
|
|
# подробный список пакетов |
|
|
|
if self.clVars.Get('cl_update_emergelist_set') == 'on': |
|
|
|
if self.clVars.Get('update.cl_update_emergelist_set') == 'on': |
|
|
|
self.printPre(self._emerge_translate( |
|
|
|
emerge.uninstall_packages.verbose_result)) |
|
|
|
else: |
|
|
@ -677,9 +735,7 @@ class Update(object): |
|
|
|
Получить список обновляемых пакетов @world из кэша |
|
|
|
""" |
|
|
|
if "@world" in packages: |
|
|
|
from calculate.update.utils.cl_update import ClUpdateAction |
|
|
|
elog = EmergeLog( |
|
|
|
EmergeLogNamedTask(ClUpdateAction.log_names['premerge'])) |
|
|
|
elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge)) |
|
|
|
if check and (elog.list or elog.remove_list): |
|
|
|
self.emerge_cache.drop_cache( |
|
|
|
"Some packages was installed or removed") |
|
|
@ -697,9 +753,7 @@ class Update(object): |
|
|
|
premerge |
|
|
|
""" |
|
|
|
self.emerge_cache.set_cache(pkg_list) |
|
|
|
from calculate.update.utils.cl_update import ClUpdateAction |
|
|
|
elog = EmergeLog( |
|
|
|
EmergeLogNamedTask(ClUpdateAction.log_names['premerge'])) |
|
|
|
elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge)) |
|
|
|
elog.mark_end_task(), |
|
|
|
|
|
|
|
def mark_schedule(self): |
|
|
@ -812,10 +866,15 @@ class Update(object): |
|
|
|
error = "<br/>".join(str(error).split('<br/>')[-lines_num:]) |
|
|
|
self.printPre(self._emerge_translate(error)) |
|
|
|
|
|
|
|
def emerge(self, param, *packages): |
|
|
|
def emerge(self, use, param, *packages): |
|
|
|
""" |
|
|
|
Выполнить сборку пакета |
|
|
|
""" |
|
|
|
if self.clVars.Get('cl_chroot_path') != '/': |
|
|
|
command_class = partial(ChrootEmergeCommand, |
|
|
|
self.clVars.Get('cl_chroot_path')) |
|
|
|
else: |
|
|
|
command_class = EmergeCommand |
|
|
|
deo = self.clVars.Get('cl_emerge_default_opts') |
|
|
|
if not packages: |
|
|
|
packages = [param] |
|
|
@ -825,8 +884,9 @@ class Update(object): |
|
|
|
if not packages: |
|
|
|
return True |
|
|
|
extra_params = [param] |
|
|
|
with EmergeParser(EmergeCommand(list(packages), emerge_default_opts=deo, |
|
|
|
extra_params=extra_params)) as emerge: |
|
|
|
with EmergeParser(command_class(list(packages), emerge_default_opts=deo, |
|
|
|
extra_params=extra_params, |
|
|
|
use=use)) as emerge: |
|
|
|
try: |
|
|
|
emerge.question.action = lambda x: False |
|
|
|
emerge.run() |
|
|
@ -911,22 +971,23 @@ class Update(object): |
|
|
|
|
|
|
|
return decor |
|
|
|
|
|
|
|
def migrateCacheRepository(self, url, branch): |
|
|
|
rep_set = self.clVars.Get('cl_update_profile_storage') |
|
|
|
rep = rep_set.get_repository(url, branch) |
|
|
|
def migrateCacheRepository(self, url, branch, storage): |
|
|
|
""" |
|
|
|
Перенести репозиторий из кэша в локальный |
|
|
|
""" |
|
|
|
rep = storage.get_repository(url, branch) |
|
|
|
if rep: |
|
|
|
rep.storage = rep_set.storages[0] |
|
|
|
rep.storage = storage.storages[0] |
|
|
|
self.clVars.Invalidate('cl_update_profile_storage') |
|
|
|
return True |
|
|
|
|
|
|
|
|
|
|
|
def reconfigureProfileVars(self): |
|
|
|
def reconfigureProfileVars(self, profile_dv, chroot): |
|
|
|
""" |
|
|
|
Синхронизировать репозитории |
|
|
|
""" |
|
|
|
dv = self.clVars |
|
|
|
|
|
|
|
profile_dv = dv.Get('cl_update_profile_datavars') |
|
|
|
try: |
|
|
|
if not profile_dv: |
|
|
|
raise UpdateError( |
|
|
@ -937,17 +998,22 @@ class Update(object): |
|
|
|
'cl_update_rep_name', |
|
|
|
'cl_update_branch_name', |
|
|
|
'cl_profile_system', |
|
|
|
'cl_update_layman_storage', |
|
|
|
'cl_update_rep'): |
|
|
|
# TODO: debug block |
|
|
|
print var_name, ":", profile_dv.Get(var_name) |
|
|
|
dv.Set(var_name, profile_dv.Get(var_name), force=True) |
|
|
|
except DataVarsError: |
|
|
|
dv.Set('cl_chroot_path', chroot, force=True) |
|
|
|
# TODO: debug block |
|
|
|
print ('cl_builder_branch_name', |
|
|
|
self.clVars.Get('cl_builder_branch_name')) |
|
|
|
except DataVarsError as e: |
|
|
|
raise UpdateError(_("Wrong profile")) |
|
|
|
return True |
|
|
|
|
|
|
|
def setProfile(self): |
|
|
|
profile = self.clVars.Select('cl_profile_path', |
|
|
|
where='cl_profile_shortname', |
|
|
|
eq=self.clVars.Get('cl_update_profile_system'), limit=1) |
|
|
|
def setProfile(self, profile_shortname): |
|
|
|
profile = self.clVars.Select('cl_update_profile_path', |
|
|
|
where='cl_update_profile_shortname', |
|
|
|
eq=profile_shortname, limit=1) |
|
|
|
if not profile: |
|
|
|
raise UpdateError(_("Failed to determine profile %s") % |
|
|
|
self.clVars.Get('cl_update_profile_system')) |
|
|
|