From 732d4cccafe3e28eb4473ce6a5b581f80b57eed1 Mon Sep 17 00:00:00 2001 From: Mike Khiretskiy Date: Fri, 31 Oct 2014 14:53:59 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D1=87=D0=B8=D1=81=D1=82=D0=BA=D0=B0=20=D0=B0?= =?UTF-8?q?=D1=80=D1=85=D0=B8=D0=B2=D0=BE=D0=B2=20=D0=B8=20=D0=B1=D0=B8?= =?UTF-8?q?=D0=BD=D0=B0=D1=80=D0=BD=D1=8B=D1=85=20=D0=BF=D0=B0=D0=BA=D0=B5?= =?UTF-8?q?=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/update/update.py | 96 +++++++++++++++++++++-------- pym/update/utils/cl_setup_update.py | 4 +- pym/update/utils/cl_update.py | 12 +++- pym/update/variables/update.py | 17 ++++- pym/update/wsdl_update.py | 5 +- 5 files changed, 103 insertions(+), 31 deletions(-) diff --git a/pym/update/update.py b/pym/update/update.py index f4d8351..e021fe7 100644 --- a/pym/update/update.py +++ b/pym/update/update.py @@ -13,14 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from itertools import ifilter import random import sys from os import path import os import time -from calculate.core.server.gen_pid import search_worked_process, ProcessStatus +from calculate.core.server.gen_pid import search_worked_process from calculate.lib.cl_template import SystemIni from calculate.lib.datavars import DataVarsError @@ -30,17 +29,22 @@ from calculate.lib.utils.colortext import get_color_print from calculate.update.emerge_parser import RevdepPercentBlock from calculate.update.datavars import DataVarsUpdate from calculate.update.update_info import UpdateInfo +from calculate.lib.cl_log import log import re +from itertools import chain from collections import MutableSet -from calculate.lib.utils.portage import Git, Layman,\ - EmergeLogNamedTask, EmergeLog, GitError, \ - PackageInformation, PackageList, EmergePackage +from calculate.lib.utils.portage import (Git, Layman, EmergeLog, GitError, + EmergeLogNamedTask, PackageList, + PackageInformation, + get_packages_files_directory, + get_manifest_files_directory, + get_remove_list) Colors = TextState.Colors from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir, PercentProgress, process, getRunCommands, - readFile, listDirectory) + readFile) import emerge_parser import logging from emerge_parser import EmergeParser, EmergeCommand, EmergeError, EmergeCache @@ -96,7 +100,7 @@ class OverlayOwnCache(MutableSet): self.__write_overlays(overlays) -class Update: +class Update(object): """Основной объект для выполнения действий связанных с обновлением системы """ @@ -121,7 +125,7 @@ class Update: """ dv = self.clVars git = Git() - needMeta = False + info_outdate = False try: self.stash_cache(rpath, name) if not git.checkExistsRep(rpath): @@ -131,7 +135,7 @@ class Update: else: git.cloneRevRepository(url, rpath, branch, revision, cb_progress=cb_progress) - needMeta = True + info_outdate = True else: # если нужно обновиться до конкретной ревизии if revision != "last": @@ -144,7 +148,7 @@ class Update: repInfo = git.getStatusInfo(rpath) if repInfo['branch'] != branch: # меняем ветку - needMeta = True + info_outdate = True git.checkoutBranch(rpath, branch) if revision == "last": if git.resetRepository(rpath, to_origin=True): @@ -152,23 +156,25 @@ class Update: repInfo = git.getStatusInfo(rpath) if repInfo.get("files", False): raise GitError("Failed to reset git") - needMeta = True + info_outdate = True else: git.resetRepository(rpath, to_rev=revision) - needMeta = True - if needMeta: + info_outdate = True + if info_outdate: dv.Set('cl_update_outdate_set', 'on', force=True) finally: self.unstash_cache(rpath, name) return True - def setAutocheckParams(self, status, interval): + def setAutocheckParams(self, status, interval, update_other, cleanpkg): """ Настроить параметры автопроверки обновлений """ - status = "on" if status else "off" - self.clVars.Write('cl_update_autocheck_set', status, True) + onoff = lambda x: "on" if x else "off" + self.clVars.Write('cl_update_autocheck_set', onoff(status), True) self.clVars.Write('cl_update_autocheck_interval', interval, True) + self.clVars.Write('cl_update_other_set', onoff(update_other), True) + self.clVars.Write('cl_update_cleanpkg_set', onoff(cleanpkg), True) return True def checkSchedule(self, interval, status): @@ -949,21 +955,20 @@ class Update: for copyvar in ("cl_dispatch_conf", "cl_verbose_set"): dv.Set(copyvar,self.clVars.Get(copyvar),True) # определение каталогов содержащих шаблоны - dirs_list, files_list = ([],[]) useClt = useClt in ("on",True) self.addProgress() nullProgress = lambda *args,**kw:None dispatch = self.dispatchConf if useDispatch else None - clTempl = ProgressTemplate(nullProgress,dv, - cltObj=useClt, - cltFilter=cltFilter, - printSUCCESS=self.printSUCCESS, - printWARNING=self.printWARNING, - askConfirm=self.askConfirm, - dispatchConf=dispatch, - printERROR=self.printERROR) + clTempl = ProgressTemplate(nullProgress, dv, + cltObj=useClt, + cltFilter=cltFilter, + printSUCCESS=self.printSUCCESS, + printWARNING=self.printWARNING, + askConfirm=self.askConfirm, + dispatchConf=dispatch, + printERROR=self.printERROR) try: - dirsFiles = clTempl.applyTemplates() + clTempl.applyTemplates() if clTempl.hasError(): if clTempl.getError(): raise TemplatesError(clTempl.getError()) @@ -975,3 +980,42 @@ class Update: finally: dv.close() return True + + def cleanpkg(self): + """ + Очистить distfiles и pkgdir от устаревших пакетов + """ + portdirs = ([self.clVars.Get('cl_portdir')] + + self.clVars.Get('cl_portdir_overlay')) + pkgfiles = get_packages_files_directory(*portdirs) + distdirfiles = get_manifest_files_directory(*portdirs) + distdir = self.clVars.Get('install.cl_distfiles_path') + pkgdir = self.clVars.Get('cl_pkgdir') + + skip_files = ["/metadata.dtd", "/Packages"] + try: + if self.clVars.Get('client.os_remote_auth'): + skip_files += ['portage_lockfile'] + except DataVarsError: + pass + + logger = log("update_cleanpkg.log", + filename="/var/log/calculate/update_cleanpkg.log", + formatter="%(asctime)s - %(clean)s - %(message)s") + + for cleantype, filelist in (("packages", + get_remove_list(pkgdir, list(pkgfiles), depth=4)), + ("distfiles", + get_remove_list(distdir, list(distdirfiles), depth=1))): + removelist = [] + for fn in filelist: + try: + if not any(fn.endswith(x) for x in skip_files): + os.unlink(fn) + removelist.append(path.basename(fn)) + except OSError: + pass + removelist_str = ",".join(removelist) + if removelist_str: + logger.info(removelist_str, extra={'clean': cleantype}) + return True diff --git a/pym/update/utils/cl_setup_update.py b/pym/update/utils/cl_setup_update.py index 5d744fa..11843fd 100644 --- a/pym/update/utils/cl_setup_update.py +++ b/pym/update/utils/cl_setup_update.py @@ -41,5 +41,7 @@ class ClSetupUpdateAction(Action): tasks = [ {'name': 'set_variables', 'method': 'Update.setAutocheckParams(cl_update_autocheck_set,' - 'cl_update_autocheck_interval)'} + 'cl_update_autocheck_interval,' + 'cl_update_other_set,' + 'cl_update_cleanpkg_set)'} ] diff --git a/pym/update/utils/cl_update.py b/pym/update/utils/cl_update.py index 4b338b1..509aed1 100644 --- a/pym/update/utils/cl_update.py +++ b/pym/update/utils/cl_update.py @@ -181,9 +181,10 @@ class ClUpdateAction(Action): 'cl_update_autocheck_set)', 'condition': lambda Get: ( Get('cl_update_autocheck_schedule_set') == 'on'), - }, + }, {'name': 'check_run', - 'method': 'Update.checkRun(cl_update_wait_another_set)'}, + 'method': 'Update.checkRun(cl_update_wait_another_set)' + }, {'name': 'reps_synchronization', 'group': __("Repositories synchronization"), 'tasks': [ @@ -229,6 +230,13 @@ class ClUpdateAction(Action): Get('cl_update_eixupdate_force') != 'skip' or Get('cl_update_eixupdate_force') == 'force')) }, + {'name': 'sync_reps:cleanpkg', + 'message': __("Removing obsolete distfiles and binary packages"), + 'method': 'Update.cleanpkg()', + 'condition': (lambda Get: Get('cl_update_cleanpkg_set') == 'on' and + Get('cl_update_outdate_set') == 'on'), + 'essential': False + }, # сообщение удачного завершения при обновлении репозиториев {'name': 'success_syncrep', 'message': __("Synchronization finished"), diff --git a/pym/update/variables/update.py b/pym/update/variables/update.py index 5782a63..3bbd583 100644 --- a/pym/update/variables/update.py +++ b/pym/update/variables/update.py @@ -282,7 +282,6 @@ class VariableClUpdateOutdateSet(ReadonlyVariable): Если обновляются прочие оверлеи - данные считаются что устарели """ type = "bool" - value = "off" def get(self): return self.Get('cl_update_other_set') @@ -1042,6 +1041,7 @@ class VariableClUpdateKernelSrcPath(ReadonlyVariable): if path.exists(module_build_path): return os.readlink(module_build_path) + class VariableClUpdateKernelPkg(ReadonlyVariable): """ Пакет текущего ядра @@ -1054,6 +1054,7 @@ class VariableClUpdateKernelPkg(ReadonlyVariable): else: return "" + class VariableClUpdateLinesLimit(Variable): """ Количество выводимых строк при ошибке @@ -1061,6 +1062,7 @@ class VariableClUpdateLinesLimit(Variable): type = "int" value = "30" + class VariableClUpdateSkipSetupSet(Variable): """ Пропустить выполнение cl-setup-system в cl-update-profile @@ -1072,3 +1074,16 @@ class VariableClUpdateSkipSetupSet(Variable): def init(self): self.label = _("Skip the system setup") self.help = _("skip the system setup") + + +class VariableClUpdateCleanpkgSet(Variable): + """ + Пропустить выполнение cl-setup-system в cl-update-profile + """ + type = "bool" + value = "off" + opt = ["--clean-pkg"] + + def init(self): + self.label = _("Clean obsolete programs archives") + self.help = _("clean obsolete programs archives") diff --git a/pym/update/wsdl_update.py b/pym/update/wsdl_update.py index 32d92e5..250af39 100644 --- a/pym/update/wsdl_update.py +++ b/pym/update/wsdl_update.py @@ -69,6 +69,7 @@ class Wsdl(WsdlBase): 'cl_update_other_set', 'cl_update_pretend_set', 'cl_update_sync_rep', + 'cl_update_cleanpkg_set', 'cl_update_emergelist_set', 'cl_update_world', 'cl_update_egencache_force', @@ -164,6 +165,8 @@ class Wsdl(WsdlBase): 'groups': [ lambda group: group(_("Updates autocheck settings"), normal=('cl_update_autocheck_set', - 'cl_update_autocheck_interval'), + 'cl_update_autocheck_interval', + 'cl_update_cleanpkg_set', + 'cl_update_other_set'), next_label=_("Save"))]}, ]