From a21d0643c91b0e5fa51abc497a0e73407c4cbf9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B8=D1=80=D0=B5=D1=86=D0=BA=D0=B8=D0=B9=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= Date: Fri, 18 Dec 2015 14:20:18 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5=D0=BD?= =?UTF-8?q?=D0=B0=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20=D1=81=20=D1=80?= =?UTF-8?q?=D0=B5=D0=BF=D0=BE=D0=B7=D0=B8=D1=82=D0=BE=D1=80=D0=B8=D1=8F?= =?UTF-8?q?=D0=BC=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/update/update.py | 188 ++++++++++++++++++++------------- pym/update/utils/cl_update.py | 7 ++ pym/update/variables/update.py | 26 +++++ pym/update/wsdl_update.py | 2 + 4 files changed, 151 insertions(+), 72 deletions(-) diff --git a/pym/update/update.py b/pym/update/update.py index 222945c..9dbcf48 100644 --- a/pym/update/update.py +++ b/pym/update/update.py @@ -38,7 +38,7 @@ import shutil from collections import MutableSet from update_tasks import EmergeMark -from calculate.lib.utils.git import Git, GitError +from calculate.lib.utils.git import Git, GitError, MTimeKeeper from calculate.lib.utils.portage import (Layman, EmergeLog, EmergeLogNamedTask, PackageList, PackageInformation, @@ -57,6 +57,7 @@ from emerge_parser import (EmergeParser, EmergeCommand, EmergeError, from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, RegexpLocalization, _) + setLocalTranslate('cl_update3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) @@ -69,6 +70,7 @@ class OverlayOwnCache(MutableSet): """ Сет оверлеев с интегрированным кэшем """ + def __init__(self, dv=None): self.dv = dv @@ -118,6 +120,7 @@ def variable_module(var_env): return f(self, *args, **kw) finally: self.clVars.defaultModule = old_env + return wrapper return variable_module_decor @@ -127,6 +130,7 @@ class Update(MethodsInterface): """Основной объект для выполнения действий связанных с обновлением системы """ + def init(self): commandLog = path.join(self.clVars.Get('core.cl_log_path'), 'lastcommand.log') @@ -146,13 +150,16 @@ class Update(MethodsInterface): return getProgPath(program_name) def _syncRepository(self, name, url, rpath, revision, - cb_progress=None): + cb_progress=None, clean=False): """ Синхронизировать репозитори """ dv = self.clVars git = Git() info_outdated = False + old_dir = "%s.old" % git._gitDir(rpath) + if path.exists(old_dir): + clean = True try: self.stash_cache(rpath, name) if not git.checkExistsRep(rpath): @@ -166,15 +173,20 @@ class Update(MethodsInterface): tag_cr = git.getCommit(rpath, revision) cr = git.getCurrentCommit(rpath) ref_type = git.reference_type(rpath, revision) - status = git.getStatusInfo(rpath) - if (tag_cr != cr or ref_type == Git.Reference.Branch or - not status or status['files']): + if git.isNeedUnpack(rpath): need_update = True + elif tag_cr != cr or ref_type == Git.Reference.Branch: + need_update = True + elif clean: + status = git.getStatusInfo(rpath) + if not status or status['files']: + need_update = True except GitError: need_update = True if need_update: git.updateTagRepository(url, rpath, revision, - cb_progress=cb_progress) + cb_progress=cb_progress, + clean=clean) new_cr = git.getCurrentCommit(rpath) if new_cr != cr: info_outdated = True @@ -237,7 +249,8 @@ class Update(MethodsInterface): Проверить повторный запуск """ update_running = lambda: any(os.getpid() != x - for x in search_worked_process('update', dv)) + for x in + search_worked_process('update', dv)) dv = self.clVars if update_running(): if not wait_update: @@ -251,12 +264,12 @@ class Update(MethodsInterface): while update_running(): time.sleep(0.3) self.resumeProcess() - time.sleep(random.random()*3) + time.sleep(random.random() * 3) self.endTask() if self.clVars.Get('cl_chroot_status') == 'off': emerge_running = lambda: any("/usr/bin/emerge" in x - for x in getRunCommands(True)) + for x in getRunCommands(True)) if emerge_running(): if not wait_update: raise UpdateError(_("Emerge is running. " @@ -268,12 +281,26 @@ class Update(MethodsInterface): self.endTask() return True + @variable_module("update") + def trimRepositories(self, repname): + """ + Синхронизировать репозитории + """ + dv = self.clVars + rpath = \ + dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1) + git = Git() + self.addProgress() + git.trimRepository(rpath, cb_progress=self.setProgress) + return True + @variable_module("update") def syncRepositories(self, repname, clean_on_error=True): """ Синхронизировать репозитории """ dv = self.clVars + check_status = dv.GetBool('update.cl_update_check_rep_set') url, rpath, revision = ( dv.Select(["cl_update_rep_url", "cl_update_rep_path", "cl_update_rep_rev"], @@ -281,6 +308,9 @@ class Update(MethodsInterface): if not url or not rpath: raise UpdateError(_("Configuration variables for repositories " "are not setup")) + git = Git() + if not git.checkUrl(url): + raise UpdateError(_("Git %s is unavailable") % url) chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) if chroot_path == '/': rpath_orig = rpath @@ -288,49 +318,58 @@ class Update(MethodsInterface): rpath_orig = rpath[len(chroot_path):] self.addProgress() - if clean_on_error: - try: - layman = Layman(dv.Get('cl_update_layman_installed'), - dv.Get('cl_update_layman_make'), - dv.Get('cl_update_layman_conf'), - prefix=chroot_path) - if repname != "portage": - layman.add(repname, url, rpath_orig) + mtime = MTimeKeeper(path.join(rpath, "profiles/updates")) + mtime.save() + try: + if clean_on_error: + try: + layman = Layman(dv.Get('cl_update_layman_installed'), + dv.Get('cl_update_layman_make'), + dv.Get('cl_update_layman_conf'), + prefix=chroot_path) + if repname != "portage": + layman.add(repname, url, rpath_orig) + if not self._syncRepository(repname, url, rpath, revision, + cb_progress=self.setProgress, + clean=check_status): + return "skip" + return True + except GitError as e: + if e.addon: + self.printWARNING(str(e.addon)) + self.printWARNING(str(e)) + self.endTask(False) + self.startTask( + _("Re-fetching the {name} repository").format( + name=repname)) + self.addProgress() + rpath_new = "%s_new" % rpath + try: + self._syncRepository(repname, url, rpath_new, revision, + cb_progress=self.setProgress, + clean=check_status) + removeDir(rpath) + shutil.move(rpath_new, rpath) + except OSError as e: + raise UpdateError(_("Failed to modify the " + "{repname} repository").format( + repname=repname) + _(": ") + str(e)) + finally: + if path.exists(rpath_new): + removeDir(rpath_new) + else: if not self._syncRepository(repname, url, rpath, revision, - cb_progress=self.setProgress): + clean=check_status): return "skip" - return True - except GitError as e: - if e.addon: - self.printWARNING(str(e.addon)) - self.printWARNING(str(e)) - self.endTask(False) - self.startTask( - _("Re-fetching the {name} repository").format(name=repname)) - self.addProgress() - rpath_new = "%s_new" % rpath - try: - self._syncRepository(repname, url, rpath_new, revision, - cb_progress=self.setProgress) - removeDir(rpath) - shutil.move(rpath_new, rpath) - except OSError as e: - raise UpdateError(_("Failed to modify the " - "{repname} repository").format( - repname=repname)+_(": ")+str(e)) - finally: - if path.exists(rpath_new): - removeDir(rpath_new) - else: - if not self._syncRepository(repname, url, rpath, revision): - return "skip" - - layman = Layman(dv.Get('cl_update_layman_installed'), - dv.Get('cl_update_layman_make'), - dv.Get('cl_update_layman_conf'), - prefix=chroot_path) - if repname != "portage": - layman.add(repname, url, rpath_orig) + + layman = Layman(dv.Get('cl_update_layman_installed'), + dv.Get('cl_update_layman_make'), + dv.Get('cl_update_layman_conf'), + prefix=chroot_path) + if repname != "portage": + layman.add(repname, url, rpath_orig) + finally: + mtime.restore() return True metadata_cache_names = ("metadata/md5-cache", "metadata/cache") @@ -366,7 +405,7 @@ class Update(MethodsInterface): if any(path.exists(path.join(rpath, x)) for x in cachenames): for cachename in cachenames: cachedir_s = path.join(path.dirname(rpath), - path.basename(cachename)+".stash") + path.basename(cachename) + ".stash") if path.exists(cachedir_s): try: removeDir(cachedir_s) @@ -377,7 +416,7 @@ class Update(MethodsInterface): for cachename in cachenames: cachedir = path.join(rpath, cachename) cachedir_s = path.join(path.dirname(rpath), - path.basename(cachename)+".stash") + path.basename(cachename) + ".stash") if path.exists(cachedir_s): try: shutil.move(cachedir_s, cachedir) @@ -410,7 +449,8 @@ class Update(MethodsInterface): p = process(layman, "-s", repname, stderr=STDOUT) if p.failed(): raise UpdateError( - _("Failed to update the {rname} repository").format(rname=repname), + _("Failed to update the {rname} repository").format( + rname=repname), addon=p.read()) finally: self.unstash_cache(rpath, laymanname) @@ -418,7 +458,7 @@ class Update(MethodsInterface): def _regenCache_process(self, progname, repname, cpu_num): return process(progname, "--repo=%s" % repname, "--update", - "--jobs=%s" % cpu_num, stderr=STDOUT) + "--jobs=%s" % cpu_num, stderr=STDOUT) def regenCache(self, repname): """ @@ -469,7 +509,7 @@ class Update(MethodsInterface): data = p.read() with open('/var/log/calculate/failed-metadata-%d.log' % time.time(), 'w') as f: - f.write(data+p.alldata) + f.write(data + p.alldata) raise UpdateError(_("Failed to update metadata"), addon=data) return True @@ -520,7 +560,7 @@ class Update(MethodsInterface): two = _print("{0}", max_num) part = _("({current} of {maximum})").format(current=one, maximum=two) _print = _print.foreground(Colors.DEFAULT) - if self.is_binary_pkg(pkg,binary): + if self.is_binary_pkg(pkg, binary): _colorprint = _print.foreground(Colors.PURPLE) else: _colorprint = _print.foreground(Colors.GREEN) @@ -544,14 +584,14 @@ class Update(MethodsInterface): """ self.endTask() _print = self.color_print - if self.is_binary_pkg(pkg,binary): + if self.is_binary_pkg(pkg, binary): _print = _print.foreground(Colors.PURPLE) else: _print = _print.foreground(Colors.GREEN) pkg_key = "{CATEGORY}/{PF}".format(**pkg) if pkg_key in self.update_map: self.startTask(_("Installing {pkg} [{oldver}]").format( - pkg=_print(str(pkg)), oldver=self.update_map[ pkg_key])) + pkg=_print(str(pkg)), oldver=self.update_map[pkg_key])) self.update_map.pop(pkg_key) else: self.startTask(_("Installing %s") % (_print(str(pkg)))) @@ -649,7 +689,7 @@ class Update(MethodsInterface): if pkg.info['DESCRIPTION']: fullname = "%s " % _(pkg.info['DESCRIPTION']) - fullname = fullname[:1].upper()+fullname[1:] + fullname = fullname[:1].upper() + fullname[1:] else: fullname = "" shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"])) @@ -660,8 +700,9 @@ class Update(MethodsInterface): mult = _print.bold("*") self.printDefault( " {mult} {fullname}{flag}{shortname}{size}".format( - mult=mult, fullname=fullname, shortname=shortname, size=size, - flag=install_flag)) + mult=mult, fullname=fullname, shortname=shortname, + size=size, + flag=install_flag)) def _display_install_package(self, emerge, emergelike=False): """ @@ -683,7 +724,7 @@ class Update(MethodsInterface): emerge.install_packages.remove_list, remove_list=True) if len(emerge.install_packages.list) > 0: install_mess = (_("{count} packages will be installed").format( - count=len(emerge.install_packages.list)) + ", ") + count=len(emerge.install_packages.list)) + ", ") else: install_mess = "" if str(emerge.download_size) != "0 kB": @@ -829,8 +870,10 @@ class Update(MethodsInterface): emerge.installing.add_observer(self._printInstallPackage) emerge.uninstalling.add_observer(self._printUninstallPackage) emerge.fetching.add_observer(self._printFetching) + def cancel_observing_fetch(fn): emerge.fetching.clear_observers() + emerge.fetching.add_observer(cancel_observing_fetch) try: emerge.run() @@ -888,7 +931,7 @@ class Update(MethodsInterface): emerge = None try: emerge = EmergeParser(EmergeCommand(["--depclean"], - emerge_default_opts=deo)) + emerge_default_opts=deo)) outdated_kernel = False try: emerge.question.action = lambda x: False @@ -903,7 +946,7 @@ class Update(MethodsInterface): "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in emerge.uninstall_packages.list if ("%s-%s" % (x['CATEGORY/PN'], - x['PVR'])) != kernel_pkg] + x['PVR'])) != kernel_pkg] emerge.command.send('n\n') emerge.close() emerge = None @@ -962,7 +1005,6 @@ class Update(MethodsInterface): self.clVars.Invalidate('cl_update_profile_storage') return True - def reconfigureProfileVars(self, profile_dv, chroot): """ Синхронизировать репозитории @@ -990,8 +1032,8 @@ class Update(MethodsInterface): def setProfile(self, profile_shortname): profile = self.clVars.Select('cl_update_profile_path', - where='cl_update_profile_shortname', - eq=profile_shortname, limit=1) + where='cl_update_profile_shortname', + eq=profile_shortname, limit=1) if not profile: raise UpdateError(_("Failed to determine profile %s") % self.clVars.Get('cl_update_profile_system')) @@ -1001,13 +1043,14 @@ class Update(MethodsInterface): if not path.exists( path.join(path.dirname(profile_file), profile_path)): raise UpdateError( - _("Failed to set the profile: %s")%_("Profile not found")) + _("Failed to set the profile: %s") % _("Profile not found")) for rm_fn in filter(path.lexists, - ('/etc/make.profile', '/etc/portage/make.profile')): + ('/etc/make.profile', + '/etc/portage/make.profile')): os.unlink(rm_fn) os.symlink(profile_path, profile_file) except (OSError, IOError) as e: - raise UpdateError(_("Failed to set the profile: %s")%str(e)) + raise UpdateError(_("Failed to set the profile: %s") % str(e)) return True def applyProfileTemplates(self, useClt=None, cltFilter=False, @@ -1016,6 +1059,7 @@ class Update(MethodsInterface): Наложить шаблоны из профиля """ from calculate.lib.cl_template import TemplatesError, ProgressTemplate + dv = DataVarsUpdate() try: dv.importUpdate() @@ -1122,7 +1166,7 @@ class Update(MethodsInterface): Проверить, что доступен хотя бы один из binhost'ов :return: """ - hosts =self.clVars.Get("update.cl_update_binhost_host") + hosts = self.clVars.Get("update.cl_update_binhost_host") if not hosts: self.clVars.Delete('cl_update_binhost', location="system") raise UpdateError("Binhost is unavailable") @@ -1169,7 +1213,7 @@ class Update(MethodsInterface): branch = dv.Get('update.cl_update_branch') revs = [ branch for x in dv.Get('update.cl_update_rep_name') - ] + ] dv.Set('update.cl_update_branch_name', revs) dv.Invalidate('update.cl_update_rep_rev') return True diff --git a/pym/update/utils/cl_update.py b/pym/update/utils/cl_update.py index 333c2fa..ca2fbb0 100644 --- a/pym/update/utils/cl_update.py +++ b/pym/update/utils/cl_update.py @@ -223,6 +223,13 @@ class ClUpdateAction(Action): 'method': 'Update.syncLaymanRepository(eachvar)', 'condition': lambda Get: Get('cl_update_other_set') == 'on' }, + {'name': 'trim_reps', + 'foreach': 'cl_update_sync_rep', + 'message': __("Cleaning the {eachvar:capitalize} repository"), + 'method': 'Update.trimRepositories(eachvar)', + 'condition': lambda Get: (Get('cl_update_sync_rep') and + Get('cl_update_onedepth_set') == 'on') + }, {'name': 'sync_reps:regen_cache', 'foreach': 'cl_update_sync_overlay_rep', 'essential': False, diff --git a/pym/update/variables/update.py b/pym/update/variables/update.py index b53f068..ddcc85b 100644 --- a/pym/update/variables/update.py +++ b/pym/update/variables/update.py @@ -1446,3 +1446,29 @@ class VariableClUpdateBinhostTimeout(Variable): """ type = "int" value = "5" + +class VariableClUpdateCheckRepSet(Variable): + """ + Удлять лишние файлы из репозиториев (например созданные пользователем) + """ + type = "bool" + value = "off" + + opt = ["--check-repos", "-C"] + + def init(self): + self.label = _("Check the repositories integrity") + self.help = _("check and fix the repositories integrity") + +class VariableClUpdateOnedepthSet(Variable): + """ + Удлять лишние файлы из репозиториев (например созданные пользователем) + """ + type = "bool" + value = "off" + + opt = ["--one-depth", "-1"] + + def init(self): + self.label = _("Clear git repositories") + self.help = _("clear the history of git repositories") diff --git a/pym/update/wsdl_update.py b/pym/update/wsdl_update.py index f0b3f4f..5cf7df7 100644 --- a/pym/update/wsdl_update.py +++ b/pym/update/wsdl_update.py @@ -78,7 +78,9 @@ class Wsdl(WsdlBase): 'cl_update_wait_another_set', 'cl_update_autocheck_schedule_set', 'cl_update_binhost_recheck_set', + 'cl_update_onedepth_set', 'cl_update_cleanpkg_set', + 'cl_update_check_rep_set', 'cl_update_branch_data', 'cl_templates_locate', 'cl_verbose_set', 'cl_dispatch_conf'),