diff --git a/pym/update/emerge_parser.py b/pym/update/emerge_parser.py index e070e15..7047cb7 100644 --- a/pym/update/emerge_parser.py +++ b/pym/update/emerge_parser.py @@ -94,7 +94,6 @@ class CommandExecutor(object): if self.child: self.child.send(s) - class ChrootCommandExecutor(CommandExecutor): """ Команда запускаемая в chroot diff --git a/pym/update/update.py b/pym/update/update.py index 6e06d32..8056e44 100644 --- a/pym/update/update.py +++ b/pym/update/update.py @@ -34,7 +34,6 @@ from calculate.update.update_info import UpdateInfo from calculate.lib.cl_log import log import re from collections import MutableSet -from mock import self from update_tasks import EmergeMark from calculate.lib.utils.portage import (Git, Layman, EmergeLog, GitError, @@ -206,11 +205,15 @@ class Update(object): # git.resetRepository(rpath, to_rev=revision) # info_outdate = True if info_outdate: + self.raiseOutdate() dv.Set('cl_update_outdate_set', 'on', force=True) finally: self.unstash_cache(rpath, name) return True + def raiseOutdate(self): + self.clVars.Set('cl_update_outdate_set', 'on', force=True) + def setAutocheckParams(self, status, interval, update_other, cleanpkg): """ Настроить параметры автопроверки обновлений @@ -305,6 +308,8 @@ class Update(object): raise UpdateError(_("Configuration variables for repositories " "are not setup")) chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) + # TODO: DEBUG + # print "DEBUG", repname, revision, branch if chroot_path == '/': rpath_orig = rpath else: @@ -417,6 +422,7 @@ class Update(object): layman = self.get_prog_path('/usr/bin/layman') if not layman: raise UpdateError(_("The Layman tool is not found")) + rpath = self.clVars.Select('cl_update_other_rep_path', where='cl_update_other_rep_name', eq=repname, limit=1) @@ -815,7 +821,7 @@ class Update(object): """ Установить флаг: есть обновления """ - if self.clVars.Get('cl_update_autocheck_set') == 'off': + if self.clVars.Get('update.cl_update_autocheck_set') == 'off': val = False UpdateInfo.set_update_ready(val) return True @@ -1000,12 +1006,12 @@ class Update(object): 'cl_profile_system', 'cl_update_rep'): # TODO: debug block - print var_name, ":", profile_dv.Get(var_name) + #print var_name, ":", profile_dv.Get(var_name) dv.Set(var_name, profile_dv.Get(var_name), force=True) dv.Set('cl_chroot_path', chroot, force=True) # TODO: debug block - print ('cl_builder_branch_name', - self.clVars.Get('cl_builder_branch_name')) + #print ('cl_builder_branch_name', + # self.clVars.Get('cl_builder_branch_name')) except DataVarsError as e: raise UpdateError(_("Wrong profile")) return True @@ -1027,29 +1033,30 @@ class Update(object): raise UpdateError(_("Failed to set the profile: %s")%str(e)) return True - def applyProfileTemplates(self,useClt=None,cltFilter=False,useDispatch=True): + def applyProfileTemplates(self, useClt=None, cltFilter=False, + useDispatch=True, action="merge"): """ Наложить шаблоны из профиля """ - from calculate.lib.cl_template import (Template,TemplatesError, - ProgressTemplate) + from calculate.lib.cl_template import TemplatesError, ProgressTemplate dv = DataVarsUpdate() try: dv.importUpdate() dv.flIniFile() - dv.Set('cl_action','merge',force=True) - dv.Set('cl_templates_locate', self.clVars.Get('cl_update_templates_locate')) - dv.Set("cl_chroot_path",'/', True) - dv.Set("cl_root_path",'/', True) - for copyvar in ("cl_dispatch_conf", "cl_verbose_set"): - dv.Set(copyvar,self.clVars.Get(copyvar),True) + dv.Set('cl_action', action, force=True) + dv.Set('cl_templates_locate', + self.clVars.Get('cl_update_templates_locate')) + dv.Set("cl_chroot_path", '/', True) + dv.Set("cl_root_path", '/', True) + for copyvar in ("cl_dispatch_conf", "cl_verbose_set", + "update.cl_update_world"): + dv.Set(copyvar, self.clVars.Get(copyvar), True) # определение каталогов содержащих шаблоны - useClt = useClt in ("on",True) + useClt = useClt in ("on", True) self.addProgress() - nullProgress = lambda *args,**kw:None + nullProgress = lambda *args, **kw: None dispatch = self.dispatchConf if useDispatch else None - clTempl = ProgressTemplate(nullProgress, dv, - cltObj=useClt, + clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt, cltFilter=cltFilter, printSUCCESS=self.printSUCCESS, printWARNING=self.printWARNING, @@ -1071,9 +1078,6 @@ class Update(object): return True def cleanpkg(self): - """ - Очистить distfiles и pkgdir от устаревших пакетов - """ portdirs = ([self.clVars.Get('cl_portdir')] + self.clVars.Get('cl_portdir_overlay')) pkgfiles = get_packages_files_directory(*portdirs) @@ -1081,6 +1085,20 @@ class Update(object): distdir = self.clVars.Get('install.cl_distfiles_path') pkgdir = self.clVars.Get('cl_pkgdir') + logger = log("update_cleanpkg.log", + filename="/var/log/calculate/update_cleanpkg.log", + formatter="%(asctime)s - %(clean)s - %(message)s") + + return self._cleanpkg( + distdir, pkgdir, distdirfiles, pkgfiles, logger) + + def _update_binhost_packages(self): + os.system('/usr/sbin/emaint binhost -f &>/dev/null') + + def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger): + """ + Очистить distfiles и pkgdir от устаревших пакетов + """ skip_files = ["/metadata.dtd", "/Packages"] try: if self.clVars.Get('client.os_remote_auth'): @@ -1088,14 +1106,11 @@ class Update(object): except DataVarsError: pass - logger = log("update_cleanpkg.log", - filename="/var/log/calculate/update_cleanpkg.log", - formatter="%(asctime)s - %(clean)s - %(message)s") - - for cleantype, filelist in (("packages", - get_remove_list(pkgdir, list(pkgfiles), depth=4)), - ("distfiles", - get_remove_list(distdir, list(distdirfiles), depth=1))): + for cleantype, filelist in ( + ("packages", + get_remove_list(pkgdir, list(pkgfiles), depth=4)), + ("distfiles", + get_remove_list(distdir, list(distdirfiles), depth=1))): removelist = [] for fn in filelist: try: @@ -1109,7 +1124,7 @@ class Update(object): logger.info(removelist_str, extra={'clean': cleantype}) if cleantype == "packages": try: - os.system('/usr/sbin/emaint binhost -f &>/dev/null') + self._update_binhost_packages() for dn in listDirectory(pkgdir, fullPath=True): if path.isdir(dn) and not listDirectory(dn): os.rmdir(dn) diff --git a/pym/update/utils/cl_update.py b/pym/update/utils/cl_update.py index 288277b..188b3d1 100644 --- a/pym/update/utils/cl_update.py +++ b/pym/update/utils/cl_update.py @@ -272,8 +272,8 @@ class ClUpdateAction(Action): ] } ] + emerge_tasks + [ - {'name':'failed', - 'error':__("Update failed"), + {'name': 'failed', + 'error': __("Update failed"), 'depend': (Tasks.failed() & Tasks.hasnot("interrupt") & (Tasks.hasnot("check_schedule") | Tasks.success_all("check_schedule")))}, diff --git a/pym/update/utils/cl_update_profile.py b/pym/update/utils/cl_update_profile.py index e37724b..3f17aa4 100644 --- a/pym/update/utils/cl_update_profile.py +++ b/pym/update/utils/cl_update_profile.py @@ -113,15 +113,16 @@ class ClUpdateProfileAction(Action): }, {'name': 'revision', 'message': __("Fixing the settings"), - 'method': 'Update.applyTemplates(install.cl_source,' - 'cl_template_clt_set,True,None,False)', + 'method': 'Update.applyProfileTemplates(cl_template_clt_set,' + 'True,False,"update_profile")', 'condition': lambda Get: Get('cl_templates_locate') }, {'name': 'reconfigure', 'message': __("The system is being configured"), - 'method': 'Update.applyProfileTemplates(cl_template_clt_set,True,False)', - 'condition': lambda Get: (Get('cl_update_templates_locate') - and Get('cl_update_skip_setup_set') == 'off') + 'method': 'Update.applyProfileTemplates(cl_template_clt_set,' + 'True,False,"merge")', + 'condition': lambda Get: (Get('cl_update_templates_locate') and + Get('cl_update_skip_setup_set') == 'off') }, {'name': 'dispatch_conf', 'message': __("Updating configuration files"), diff --git a/pym/update/variables/update.py b/pym/update/variables/update.py index 3185cb3..8b539d4 100644 --- a/pym/update/variables/update.py +++ b/pym/update/variables/update.py @@ -35,6 +35,8 @@ from ..profile import RepositoryStorageSet, DEFAULT_BRANCH, \ from calculate.lib.variables import linux as lib_linux from calculate.lib.variables import env from calculate.update.update_info import UpdateInfo +import urllib2 +import time from functools import partial _ = lambda x:x @@ -165,11 +167,13 @@ class VariableClUpdateRepPath(ReadonlyVariable): Пути до репозиториев """ type = "list" - mapPath = {'portage':'usr/portage'} + mapPath = {'portage': 'usr/portage', + 'gentoo': '/usr/portage'} def get(self): repPath = self.Get('cl_update_layman_storage') chroot_path = self.Get('cl_chroot_path') + def generatePaths(names): for name in names: if name in self.mapPath: @@ -217,8 +221,7 @@ class VariableClUpdateBranch(TableVariable): self.help = _("set branches for repository (REPOSITORY:BRANCH)") self.label = _("Repositories branches") - def raiseReadonlyIndexError(self,fieldname="",variablename="", - value=""): + def raiseReadonlyIndexError(self, fieldname="", variablename="", value=""): """ Неизвестный оврелей """ @@ -236,6 +239,94 @@ class VariableClUpdateBranchRep(ReadonlyVariable): def get(self): return self.Get('cl_update_rep_name') +class VariableClUpdateBinhostData(ReadonlyTableVariable): + """ + Содержимое файла revisions + """ + source = ["cl_update_binhost_host", + "cl_update_binhost_revisions", + "cl_update_binhost_time"] + + def check_binhost(self, binhost): + revision_files = [path.join(binhost, x) + for x in self.Get('cl_update_revision_path')] + timeout = self.GetInteger('cl_update_binhost_timeout') + try: + data = None + t = time.time() + for fn in revision_files: + if data is None: + data = urllib2.urlopen(fn, timeout=timeout).read() + elif data != urllib2.urlopen(fn, timeout=timeout).read(): + return "", str(-1) + return data, str(int((time.time() - t)*1000)) + except urllib2.URLError: + return "", str(-1) + + re_revison = re.compile("\w+=(\w+)") + + def binhost_key(self, data): + host, data, time = data + try: + cp = ConfigParser() + cp.read_string(data.decode('utf-8')) + data = sum(int(x) for x in cp['vcs'].values()) + except (TypeError, KeyError) as e: + data = 0 + return (1 if int(time) >= 0 else 0, + data, + -int(time)) + + def get(self, hr=False): + def generate(): + recheck = self.GetBool('cl_update_binhost_recheck_set') + base_dn = self.Get('cl_update_binhost') + for host in [base_dn] + [x for x in self.Get('cl_update_binhosts') + if x != base_dn]: + if host: + data, t = self.check_binhost(host) + yield host, data, str(t) + if not recheck and t >= 0 and base_dn == host: + break + return list(sorted(generate(), key=self.binhost_key, reverse=True)) + + +class VariableClUpdateBinhostRecheckSet(Variable): + """ + Принудительно обновить binhost + """ + value = "off" + +class VariableClUpdateBinhostHost(FieldValue,ReadonlyVariable): + """ + Список имен прочих репозиториев + """ + type = "list" + source_variable = "cl_update_binhost_data" + column = 0 + +class VariableClUpdateBinhostRevisions(FieldValue,ReadonlyVariable): + """ + Список имен прочих репозиториев + """ + type = "list" + source_variable = "cl_update_binhost_data" + column = 1 + +class VariableClUpdateBinhostTime(FieldValue,ReadonlyVariable): + """ + Список имен прочих репозиториев + """ + type = "list" + source_variable = "cl_update_binhost_data" + column = 2 + +class VariableClUpdateRepoTagSet(Variable): + """ + Флаг принудительного использования тэгов вместо веток + """ + type = "bool" + value = "on" class VariableClUpdateBranchName(Variable): """ @@ -250,13 +341,24 @@ class VariableClUpdateBranchName(Variable): return ["master", "develop", "update"] def get(self): + cp = ConfigParser() + cp.read_string(unicode( + self.Get('update.cl_update_binhost_revisions')[0])) + repo_tag = self.GetBool('update.cl_update_repo_tag_set') + def generateBranch(): git = Git() - for reppath in self.Get('cl_update_rep_path'): + for reppath, repname in zip(self.Get('cl_update_rep_path'), + self.Get('cl_update_rep_name')): + tag = cp.get("vcs", repname, fallback="master") try: - yield git.getBranch(reppath) or "master" + if (repo_tag or git.isTagRepository( + reppath) or not git.is_git(reppath)): + yield tag + else: + yield git.getBranch(reppath) or tag except GitError: - yield "master" + yield tag return list(generateBranch()) @@ -271,20 +373,24 @@ class VariableClUpdateSyncRep(Variable): metavalue = "REPOSITORIES" untrusted = True + @property + def rep_name(self): + return self.Get('update.cl_update_rep_name') + def init(self): self.help = _("synchronized repositories (all by default)") self.label = _("Synchronized repositories") - def set(self,value): - orderList = self.Get('cl_update_rep_name') - return sorted(value,key=lambda x: - (orderList.index(x) if x in orderList else -1),reverse=True) + def set(self, value): + orderList = self.rep_name + return sorted(value, key=lambda x: + (orderList.index(x) if x in orderList else -1), reverse=True) def get(self): - return list(reversed(self.Get('cl_update_rep_name'))) + return list(reversed(self.rep_name)) def choice(self): - return self.Get('cl_update_rep_name') + return self.rep_name class VariableClUpdateSyncOverlayRep(ReadonlyVariable): """ @@ -293,7 +399,7 @@ class VariableClUpdateSyncOverlayRep(ReadonlyVariable): type = "list" def get(self): - return filter(lambda x:x!="portage",self.Get('cl_update_sync_rep')) + return filter(lambda x: x != "portage", self.Get('cl_update_sync_rep')) class VariableClUpdateOutdateSet(ReadonlyVariable): """ @@ -399,16 +505,18 @@ class VariableClUpdateOtherRepData(ReadonlyTableVariable): source = ['cl_update_other_rep_name', 'cl_update_other_rep_path'] + portdir_overlay = "main.cl_portdir_overlay" + def generator(self): - repNames = self.Get('cl_update_rep_name') - layman = Layman(self.Get('cl_update_layman_installed'), - self.Get('cl_update_layman_make'), - self.Get('cl_update_layman_conf')) + repNames = self.Get('update.cl_update_rep_name') + layman = Layman(self.Get('update.cl_update_layman_installed'), + self.Get('update.cl_update_layman_make'), + self.Get('update.cl_update_layman_conf')) layman_overlays = layman.get_installed() - for rpath in self.Get('cl_portdir_overlay'): + for rpath in self.Get(self.portdir_overlay): repo_file = path.join(rpath, "profiles/repo_name") rname = readFile(repo_file).strip() or path.basename(rpath) - if rname in layman_overlays and not rname in repNames: + if rname in layman_overlays and rname not in repNames: yield (rname, rpath) def get(self): @@ -1043,31 +1151,45 @@ class DataVarsUpdateProfile(SimpleDataVars): 'cl_update_rep_path', 'cl_update_rep_rev', 'cl_update_branch_name'] - def __init__(self, profile, chroot_path='/'): - SimpleDataVars.__init__(self, - lib_linux.VariableOsLinuxName(), - lib_linux.VariableOsLinuxShortname(), - lib_linux.VariableOsLinuxSubname(), - lib_linux.VariableOsLinuxVer(), - lib_linux.VariableClProfileSystem(), - env.VariableClRepositoryData(), - env.VariableClRepositoryName(), - env.VariableClRepositoryLocation(), - env.VariableClChrootPath(), - env.VariableClTemplateLocation(), - env.VariableClTemplatePath(), - env.VariableClEmergeConfig(systemRoot=chroot_path), - VariableClUpdateRepData(section="update"), - VariableClUpdateRepPath(section="update"), - VariableClUpdateRepRev(section="update"), - VariableClUpdateBranchName(section="update"), - VariableClUpdateLaymanConfig(section="update"), - VariableClUpdateLaymanStorage(section="update"), - VariableClUpdateRepName(section="update"), - VariableClUpdateRep(section="update"), - VariableClUpdateRepUrl(section="update")) + def __init__(self, profile, chroot_path='/', recheck=None): + SimpleDataVars.__init__( + self, + lib_linux.VariableOsLinuxName(), + lib_linux.VariableOsLinuxShortname(), + lib_linux.VariableOsLinuxSubname(), + lib_linux.VariableOsLinuxVer(), + lib_linux.VariableClProfileSystem(), + env.VariableClRepositoryData(), + env.VariableClRepositoryName(), + env.VariableClRepositoryLocation(), + env.VariableClChrootPath(), + env.VariableClTemplateLocation(), + env.VariableClTemplatePath(), + env.VariableClEmergeConfig(systemRoot=chroot_path), + VariableClUpdateRepData(section="update"), + VariableClUpdateRepPath(section="update"), + VariableClUpdateRepRev(section="update"), + VariableClUpdateBranchName(section="update"), + VariableClUpdateLaymanConfig(section="update"), + VariableClUpdateLaymanStorage(section="update"), + VariableClUpdateRepName(section="update"), + VariableClUpdateRep(section="update"), + VariableClUpdateRepUrl(section="update"), + VariableClUpdateBinhost(section="update"), + VariableClUpdateBinhostData(section="update"), + VariableClUpdateBinhostHost(section="update"), + VariableClUpdateBinhostRecheckSet(section="update"), + VariableClUpdateBinhostRevisions(section="update"), + VariableClUpdateBinhostTime(section="update"), + VariableClUpdateBinhostTimeout(section="update"), + VariableClUpdateBinhosts(section="update"), + VariableClUpdateRepoTagSet(section="update"), + VariableClUpdateRevisionPath(section="update"), + ) self.cache['cl_profile_system'] = profile self.cache['cl_chroot_path'] = chroot_path + if recheck is not None: + self.cache['cl_update_binhost_recheck_set'] = recheck self.flIniFileFrom(profile) def __repr__(self): @@ -1247,3 +1369,40 @@ class VariableClUpdateOutdatedKernelSet(ReadonlyVariable): def get(self): ui = UpdateInfo(self.parent) return "on" if ui.outdated_kernel else "off" + +class VariableClUpdateBinhosts(Variable): + """ + Список хостов с бинарными обновлениями + """ + type = "list" + value = ["ftp://ftp.calculate.ru/pub", 'ftp://localhost/pub'] + +class VariableClUpdateBinhost(Variable): + """ + Хост с бинарными обновлениями + """ + def get(self): + return "" + #binhosts = self.Get('cl_update_binhosts') + #if binhosts: + # return self.Get('cl_update_binhosts')[0] + #return "" + +class VariableClUpdateRevisionPath(Variable): + """ + Путь до revisions файлов + """ + type = "list" + value = [ + "calculate/grp/default/ini.env", + "calculate/grp/kde/ini.env", + "calculate/grp/server/ini.env", + "calculate/grp/x/ini.env" + ] + +class VariableClUpdateBinhostTimeout(Variable): + """ + Таймаут на проверку одного binhost + """ + type = "int" + value = "15"