Улучшена работа с репозиториями

master3.4
parent d0a23b0528
commit a21d0643c9

@ -38,7 +38,7 @@ import shutil
from collections import MutableSet from collections import MutableSet
from update_tasks import EmergeMark from update_tasks import EmergeMark
from calculate.lib.utils.git import Git, GitError from calculate.lib.utils.git import Git, GitError, MTimeKeeper
from calculate.lib.utils.portage import (Layman, EmergeLog, from calculate.lib.utils.portage import (Layman, EmergeLog,
EmergeLogNamedTask, PackageList, EmergeLogNamedTask, PackageList,
PackageInformation, PackageInformation,
@ -57,6 +57,7 @@ from emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
RegexpLocalization, _) RegexpLocalization, _)
setLocalTranslate('cl_update3', sys.modules[__name__]) setLocalTranslate('cl_update3', sys.modules[__name__])
__ = getLazyLocalTranslate(_) __ = getLazyLocalTranslate(_)
@ -69,6 +70,7 @@ class OverlayOwnCache(MutableSet):
""" """
Сет оверлеев с интегрированным кэшем Сет оверлеев с интегрированным кэшем
""" """
def __init__(self, dv=None): def __init__(self, dv=None):
self.dv = dv self.dv = dv
@ -118,6 +120,7 @@ def variable_module(var_env):
return f(self, *args, **kw) return f(self, *args, **kw)
finally: finally:
self.clVars.defaultModule = old_env self.clVars.defaultModule = old_env
return wrapper return wrapper
return variable_module_decor return variable_module_decor
@ -127,6 +130,7 @@ class Update(MethodsInterface):
"""Основной объект для выполнения действий связанных с обновлением системы """Основной объект для выполнения действий связанных с обновлением системы
""" """
def init(self): def init(self):
commandLog = path.join(self.clVars.Get('core.cl_log_path'), commandLog = path.join(self.clVars.Get('core.cl_log_path'),
'lastcommand.log') 'lastcommand.log')
@ -146,13 +150,16 @@ class Update(MethodsInterface):
return getProgPath(program_name) return getProgPath(program_name)
def _syncRepository(self, name, url, rpath, revision, def _syncRepository(self, name, url, rpath, revision,
cb_progress=None): cb_progress=None, clean=False):
""" """
Синхронизировать репозитори Синхронизировать репозитори
""" """
dv = self.clVars dv = self.clVars
git = Git() git = Git()
info_outdated = False info_outdated = False
old_dir = "%s.old" % git._gitDir(rpath)
if path.exists(old_dir):
clean = True
try: try:
self.stash_cache(rpath, name) self.stash_cache(rpath, name)
if not git.checkExistsRep(rpath): if not git.checkExistsRep(rpath):
@ -166,15 +173,20 @@ class Update(MethodsInterface):
tag_cr = git.getCommit(rpath, revision) tag_cr = git.getCommit(rpath, revision)
cr = git.getCurrentCommit(rpath) cr = git.getCurrentCommit(rpath)
ref_type = git.reference_type(rpath, revision) ref_type = git.reference_type(rpath, revision)
status = git.getStatusInfo(rpath) if git.isNeedUnpack(rpath):
if (tag_cr != cr or ref_type == Git.Reference.Branch or
not status or status['files']):
need_update = True need_update = True
elif tag_cr != cr or ref_type == Git.Reference.Branch:
need_update = True
elif clean:
status = git.getStatusInfo(rpath)
if not status or status['files']:
need_update = True
except GitError: except GitError:
need_update = True need_update = True
if need_update: if need_update:
git.updateTagRepository(url, rpath, revision, git.updateTagRepository(url, rpath, revision,
cb_progress=cb_progress) cb_progress=cb_progress,
clean=clean)
new_cr = git.getCurrentCommit(rpath) new_cr = git.getCurrentCommit(rpath)
if new_cr != cr: if new_cr != cr:
info_outdated = True info_outdated = True
@ -237,7 +249,8 @@ class Update(MethodsInterface):
Проверить повторный запуск Проверить повторный запуск
""" """
update_running = lambda: any(os.getpid() != x update_running = lambda: any(os.getpid() != x
for x in search_worked_process('update', dv)) for x in
search_worked_process('update', dv))
dv = self.clVars dv = self.clVars
if update_running(): if update_running():
if not wait_update: if not wait_update:
@ -251,12 +264,12 @@ class Update(MethodsInterface):
while update_running(): while update_running():
time.sleep(0.3) time.sleep(0.3)
self.resumeProcess() self.resumeProcess()
time.sleep(random.random()*3) time.sleep(random.random() * 3)
self.endTask() self.endTask()
if self.clVars.Get('cl_chroot_status') == 'off': if self.clVars.Get('cl_chroot_status') == 'off':
emerge_running = lambda: any("/usr/bin/emerge" in x emerge_running = lambda: any("/usr/bin/emerge" in x
for x in getRunCommands(True)) for x in getRunCommands(True))
if emerge_running(): if emerge_running():
if not wait_update: if not wait_update:
raise UpdateError(_("Emerge is running. " raise UpdateError(_("Emerge is running. "
@ -268,12 +281,26 @@ class Update(MethodsInterface):
self.endTask() self.endTask()
return True return True
@variable_module("update")
def trimRepositories(self, repname):
"""
Синхронизировать репозитории
"""
dv = self.clVars
rpath = \
dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1)
git = Git()
self.addProgress()
git.trimRepository(rpath, cb_progress=self.setProgress)
return True
@variable_module("update") @variable_module("update")
def syncRepositories(self, repname, clean_on_error=True): def syncRepositories(self, repname, clean_on_error=True):
""" """
Синхронизировать репозитории Синхронизировать репозитории
""" """
dv = self.clVars dv = self.clVars
check_status = dv.GetBool('update.cl_update_check_rep_set')
url, rpath, revision = ( url, rpath, revision = (
dv.Select(["cl_update_rep_url", "cl_update_rep_path", dv.Select(["cl_update_rep_url", "cl_update_rep_path",
"cl_update_rep_rev"], "cl_update_rep_rev"],
@ -281,6 +308,9 @@ class Update(MethodsInterface):
if not url or not rpath: if not url or not rpath:
raise UpdateError(_("Configuration variables for repositories " raise UpdateError(_("Configuration variables for repositories "
"are not setup")) "are not setup"))
git = Git()
if not git.checkUrl(url):
raise UpdateError(_("Git %s is unavailable") % url)
chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
if chroot_path == '/': if chroot_path == '/':
rpath_orig = rpath rpath_orig = rpath
@ -288,49 +318,58 @@ class Update(MethodsInterface):
rpath_orig = rpath[len(chroot_path):] rpath_orig = rpath[len(chroot_path):]
self.addProgress() self.addProgress()
if clean_on_error: mtime = MTimeKeeper(path.join(rpath, "profiles/updates"))
try: mtime.save()
layman = Layman(dv.Get('cl_update_layman_installed'), try:
dv.Get('cl_update_layman_make'), if clean_on_error:
dv.Get('cl_update_layman_conf'), try:
prefix=chroot_path) layman = Layman(dv.Get('cl_update_layman_installed'),
if repname != "portage": dv.Get('cl_update_layman_make'),
layman.add(repname, url, rpath_orig) dv.Get('cl_update_layman_conf'),
prefix=chroot_path)
if repname != "portage":
layman.add(repname, url, rpath_orig)
if not self._syncRepository(repname, url, rpath, revision,
cb_progress=self.setProgress,
clean=check_status):
return "skip"
return True
except GitError as e:
if e.addon:
self.printWARNING(str(e.addon))
self.printWARNING(str(e))
self.endTask(False)
self.startTask(
_("Re-fetching the {name} repository").format(
name=repname))
self.addProgress()
rpath_new = "%s_new" % rpath
try:
self._syncRepository(repname, url, rpath_new, revision,
cb_progress=self.setProgress,
clean=check_status)
removeDir(rpath)
shutil.move(rpath_new, rpath)
except OSError as e:
raise UpdateError(_("Failed to modify the "
"{repname} repository").format(
repname=repname) + _(": ") + str(e))
finally:
if path.exists(rpath_new):
removeDir(rpath_new)
else:
if not self._syncRepository(repname, url, rpath, revision, if not self._syncRepository(repname, url, rpath, revision,
cb_progress=self.setProgress): clean=check_status):
return "skip" return "skip"
return True
except GitError as e: layman = Layman(dv.Get('cl_update_layman_installed'),
if e.addon: dv.Get('cl_update_layman_make'),
self.printWARNING(str(e.addon)) dv.Get('cl_update_layman_conf'),
self.printWARNING(str(e)) prefix=chroot_path)
self.endTask(False) if repname != "portage":
self.startTask( layman.add(repname, url, rpath_orig)
_("Re-fetching the {name} repository").format(name=repname)) finally:
self.addProgress() mtime.restore()
rpath_new = "%s_new" % rpath
try:
self._syncRepository(repname, url, rpath_new, revision,
cb_progress=self.setProgress)
removeDir(rpath)
shutil.move(rpath_new, rpath)
except OSError as e:
raise UpdateError(_("Failed to modify the "
"{repname} repository").format(
repname=repname)+_(": ")+str(e))
finally:
if path.exists(rpath_new):
removeDir(rpath_new)
else:
if not self._syncRepository(repname, url, rpath, revision):
return "skip"
layman = Layman(dv.Get('cl_update_layman_installed'),
dv.Get('cl_update_layman_make'),
dv.Get('cl_update_layman_conf'),
prefix=chroot_path)
if repname != "portage":
layman.add(repname, url, rpath_orig)
return True return True
metadata_cache_names = ("metadata/md5-cache", "metadata/cache") metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
@ -366,7 +405,7 @@ class Update(MethodsInterface):
if any(path.exists(path.join(rpath, x)) for x in cachenames): if any(path.exists(path.join(rpath, x)) for x in cachenames):
for cachename in cachenames: for cachename in cachenames:
cachedir_s = path.join(path.dirname(rpath), cachedir_s = path.join(path.dirname(rpath),
path.basename(cachename)+".stash") path.basename(cachename) + ".stash")
if path.exists(cachedir_s): if path.exists(cachedir_s):
try: try:
removeDir(cachedir_s) removeDir(cachedir_s)
@ -377,7 +416,7 @@ class Update(MethodsInterface):
for cachename in cachenames: for cachename in cachenames:
cachedir = path.join(rpath, cachename) cachedir = path.join(rpath, cachename)
cachedir_s = path.join(path.dirname(rpath), cachedir_s = path.join(path.dirname(rpath),
path.basename(cachename)+".stash") path.basename(cachename) + ".stash")
if path.exists(cachedir_s): if path.exists(cachedir_s):
try: try:
shutil.move(cachedir_s, cachedir) shutil.move(cachedir_s, cachedir)
@ -410,7 +449,8 @@ class Update(MethodsInterface):
p = process(layman, "-s", repname, stderr=STDOUT) p = process(layman, "-s", repname, stderr=STDOUT)
if p.failed(): if p.failed():
raise UpdateError( raise UpdateError(
_("Failed to update the {rname} repository").format(rname=repname), _("Failed to update the {rname} repository").format(
rname=repname),
addon=p.read()) addon=p.read())
finally: finally:
self.unstash_cache(rpath, laymanname) self.unstash_cache(rpath, laymanname)
@ -418,7 +458,7 @@ class Update(MethodsInterface):
def _regenCache_process(self, progname, repname, cpu_num): def _regenCache_process(self, progname, repname, cpu_num):
return process(progname, "--repo=%s" % repname, "--update", return process(progname, "--repo=%s" % repname, "--update",
"--jobs=%s" % cpu_num, stderr=STDOUT) "--jobs=%s" % cpu_num, stderr=STDOUT)
def regenCache(self, repname): def regenCache(self, repname):
""" """
@ -469,7 +509,7 @@ class Update(MethodsInterface):
data = p.read() data = p.read()
with open('/var/log/calculate/failed-metadata-%d.log' % time.time(), with open('/var/log/calculate/failed-metadata-%d.log' % time.time(),
'w') as f: 'w') as f:
f.write(data+p.alldata) f.write(data + p.alldata)
raise UpdateError(_("Failed to update metadata"), addon=data) raise UpdateError(_("Failed to update metadata"), addon=data)
return True return True
@ -520,7 +560,7 @@ class Update(MethodsInterface):
two = _print("{0}", max_num) two = _print("{0}", max_num)
part = _("({current} of {maximum})").format(current=one, maximum=two) part = _("({current} of {maximum})").format(current=one, maximum=two)
_print = _print.foreground(Colors.DEFAULT) _print = _print.foreground(Colors.DEFAULT)
if self.is_binary_pkg(pkg,binary): if self.is_binary_pkg(pkg, binary):
_colorprint = _print.foreground(Colors.PURPLE) _colorprint = _print.foreground(Colors.PURPLE)
else: else:
_colorprint = _print.foreground(Colors.GREEN) _colorprint = _print.foreground(Colors.GREEN)
@ -544,14 +584,14 @@ class Update(MethodsInterface):
""" """
self.endTask() self.endTask()
_print = self.color_print _print = self.color_print
if self.is_binary_pkg(pkg,binary): if self.is_binary_pkg(pkg, binary):
_print = _print.foreground(Colors.PURPLE) _print = _print.foreground(Colors.PURPLE)
else: else:
_print = _print.foreground(Colors.GREEN) _print = _print.foreground(Colors.GREEN)
pkg_key = "{CATEGORY}/{PF}".format(**pkg) pkg_key = "{CATEGORY}/{PF}".format(**pkg)
if pkg_key in self.update_map: if pkg_key in self.update_map:
self.startTask(_("Installing {pkg} [{oldver}]").format( self.startTask(_("Installing {pkg} [{oldver}]").format(
pkg=_print(str(pkg)), oldver=self.update_map[ pkg_key])) pkg=_print(str(pkg)), oldver=self.update_map[pkg_key]))
self.update_map.pop(pkg_key) self.update_map.pop(pkg_key)
else: else:
self.startTask(_("Installing %s") % (_print(str(pkg)))) self.startTask(_("Installing %s") % (_print(str(pkg))))
@ -649,7 +689,7 @@ class Update(MethodsInterface):
if pkg.info['DESCRIPTION']: if pkg.info['DESCRIPTION']:
fullname = "%s " % _(pkg.info['DESCRIPTION']) fullname = "%s " % _(pkg.info['DESCRIPTION'])
fullname = fullname[:1].upper()+fullname[1:] fullname = fullname[:1].upper() + fullname[1:]
else: else:
fullname = "" fullname = ""
shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"])) shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
@ -660,8 +700,9 @@ class Update(MethodsInterface):
mult = _print.bold("*") mult = _print.bold("*")
self.printDefault( self.printDefault(
" {mult} {fullname}{flag}{shortname}{size}".format( " {mult} {fullname}{flag}{shortname}{size}".format(
mult=mult, fullname=fullname, shortname=shortname, size=size, mult=mult, fullname=fullname, shortname=shortname,
flag=install_flag)) size=size,
flag=install_flag))
def _display_install_package(self, emerge, emergelike=False): def _display_install_package(self, emerge, emergelike=False):
""" """
@ -683,7 +724,7 @@ class Update(MethodsInterface):
emerge.install_packages.remove_list, remove_list=True) emerge.install_packages.remove_list, remove_list=True)
if len(emerge.install_packages.list) > 0: if len(emerge.install_packages.list) > 0:
install_mess = (_("{count} packages will be installed").format( install_mess = (_("{count} packages will be installed").format(
count=len(emerge.install_packages.list)) + ", ") count=len(emerge.install_packages.list)) + ", ")
else: else:
install_mess = "" install_mess = ""
if str(emerge.download_size) != "0 kB": if str(emerge.download_size) != "0 kB":
@ -829,8 +870,10 @@ class Update(MethodsInterface):
emerge.installing.add_observer(self._printInstallPackage) emerge.installing.add_observer(self._printInstallPackage)
emerge.uninstalling.add_observer(self._printUninstallPackage) emerge.uninstalling.add_observer(self._printUninstallPackage)
emerge.fetching.add_observer(self._printFetching) emerge.fetching.add_observer(self._printFetching)
def cancel_observing_fetch(fn): def cancel_observing_fetch(fn):
emerge.fetching.clear_observers() emerge.fetching.clear_observers()
emerge.fetching.add_observer(cancel_observing_fetch) emerge.fetching.add_observer(cancel_observing_fetch)
try: try:
emerge.run() emerge.run()
@ -888,7 +931,7 @@ class Update(MethodsInterface):
emerge = None emerge = None
try: try:
emerge = EmergeParser(EmergeCommand(["--depclean"], emerge = EmergeParser(EmergeCommand(["--depclean"],
emerge_default_opts=deo)) emerge_default_opts=deo))
outdated_kernel = False outdated_kernel = False
try: try:
emerge.question.action = lambda x: False emerge.question.action = lambda x: False
@ -903,7 +946,7 @@ class Update(MethodsInterface):
"=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in
emerge.uninstall_packages.list emerge.uninstall_packages.list
if ("%s-%s" % (x['CATEGORY/PN'], if ("%s-%s" % (x['CATEGORY/PN'],
x['PVR'])) != kernel_pkg] x['PVR'])) != kernel_pkg]
emerge.command.send('n\n') emerge.command.send('n\n')
emerge.close() emerge.close()
emerge = None emerge = None
@ -962,7 +1005,6 @@ class Update(MethodsInterface):
self.clVars.Invalidate('cl_update_profile_storage') self.clVars.Invalidate('cl_update_profile_storage')
return True return True
def reconfigureProfileVars(self, profile_dv, chroot): def reconfigureProfileVars(self, profile_dv, chroot):
""" """
Синхронизировать репозитории Синхронизировать репозитории
@ -990,8 +1032,8 @@ class Update(MethodsInterface):
def setProfile(self, profile_shortname): def setProfile(self, profile_shortname):
profile = self.clVars.Select('cl_update_profile_path', profile = self.clVars.Select('cl_update_profile_path',
where='cl_update_profile_shortname', where='cl_update_profile_shortname',
eq=profile_shortname, limit=1) eq=profile_shortname, limit=1)
if not profile: if not profile:
raise UpdateError(_("Failed to determine profile %s") % raise UpdateError(_("Failed to determine profile %s") %
self.clVars.Get('cl_update_profile_system')) self.clVars.Get('cl_update_profile_system'))
@ -1001,13 +1043,14 @@ class Update(MethodsInterface):
if not path.exists( if not path.exists(
path.join(path.dirname(profile_file), profile_path)): path.join(path.dirname(profile_file), profile_path)):
raise UpdateError( raise UpdateError(
_("Failed to set the profile: %s")%_("Profile not found")) _("Failed to set the profile: %s") % _("Profile not found"))
for rm_fn in filter(path.lexists, for rm_fn in filter(path.lexists,
('/etc/make.profile', '/etc/portage/make.profile')): ('/etc/make.profile',
'/etc/portage/make.profile')):
os.unlink(rm_fn) os.unlink(rm_fn)
os.symlink(profile_path, profile_file) os.symlink(profile_path, profile_file)
except (OSError, IOError) as e: except (OSError, IOError) as e:
raise UpdateError(_("Failed to set the profile: %s")%str(e)) raise UpdateError(_("Failed to set the profile: %s") % str(e))
return True return True
def applyProfileTemplates(self, useClt=None, cltFilter=False, def applyProfileTemplates(self, useClt=None, cltFilter=False,
@ -1016,6 +1059,7 @@ class Update(MethodsInterface):
Наложить шаблоны из профиля Наложить шаблоны из профиля
""" """
from calculate.lib.cl_template import TemplatesError, ProgressTemplate from calculate.lib.cl_template import TemplatesError, ProgressTemplate
dv = DataVarsUpdate() dv = DataVarsUpdate()
try: try:
dv.importUpdate() dv.importUpdate()
@ -1122,7 +1166,7 @@ class Update(MethodsInterface):
Проверить, что доступен хотя бы один из binhost'ов Проверить, что доступен хотя бы один из binhost'ов
:return: :return:
""" """
hosts =self.clVars.Get("update.cl_update_binhost_host") hosts = self.clVars.Get("update.cl_update_binhost_host")
if not hosts: if not hosts:
self.clVars.Delete('cl_update_binhost', location="system") self.clVars.Delete('cl_update_binhost', location="system")
raise UpdateError("Binhost is unavailable") raise UpdateError("Binhost is unavailable")
@ -1169,7 +1213,7 @@ class Update(MethodsInterface):
branch = dv.Get('update.cl_update_branch') branch = dv.Get('update.cl_update_branch')
revs = [ revs = [
branch for x in dv.Get('update.cl_update_rep_name') branch for x in dv.Get('update.cl_update_rep_name')
] ]
dv.Set('update.cl_update_branch_name', revs) dv.Set('update.cl_update_branch_name', revs)
dv.Invalidate('update.cl_update_rep_rev') dv.Invalidate('update.cl_update_rep_rev')
return True return True

@ -223,6 +223,13 @@ class ClUpdateAction(Action):
'method': 'Update.syncLaymanRepository(eachvar)', 'method': 'Update.syncLaymanRepository(eachvar)',
'condition': lambda Get: Get('cl_update_other_set') == 'on' 'condition': lambda Get: Get('cl_update_other_set') == 'on'
}, },
{'name': 'trim_reps',
'foreach': 'cl_update_sync_rep',
'message': __("Cleaning the {eachvar:capitalize} repository"),
'method': 'Update.trimRepositories(eachvar)',
'condition': lambda Get: (Get('cl_update_sync_rep') and
Get('cl_update_onedepth_set') == 'on')
},
{'name': 'sync_reps:regen_cache', {'name': 'sync_reps:regen_cache',
'foreach': 'cl_update_sync_overlay_rep', 'foreach': 'cl_update_sync_overlay_rep',
'essential': False, 'essential': False,

@ -1446,3 +1446,29 @@ class VariableClUpdateBinhostTimeout(Variable):
""" """
type = "int" type = "int"
value = "5" value = "5"
class VariableClUpdateCheckRepSet(Variable):
"""
Удлять лишние файлы из репозиториев (например созданные пользователем)
"""
type = "bool"
value = "off"
opt = ["--check-repos", "-C"]
def init(self):
self.label = _("Check the repositories integrity")
self.help = _("check and fix the repositories integrity")
class VariableClUpdateOnedepthSet(Variable):
"""
Удлять лишние файлы из репозиториев (например созданные пользователем)
"""
type = "bool"
value = "off"
opt = ["--one-depth", "-1"]
def init(self):
self.label = _("Clear git repositories")
self.help = _("clear the history of git repositories")

@ -78,7 +78,9 @@ class Wsdl(WsdlBase):
'cl_update_wait_another_set', 'cl_update_wait_another_set',
'cl_update_autocheck_schedule_set', 'cl_update_autocheck_schedule_set',
'cl_update_binhost_recheck_set', 'cl_update_binhost_recheck_set',
'cl_update_onedepth_set',
'cl_update_cleanpkg_set', 'cl_update_cleanpkg_set',
'cl_update_check_rep_set',
'cl_update_branch_data', 'cl_update_branch_data',
'cl_templates_locate', 'cl_templates_locate',
'cl_verbose_set', 'cl_dispatch_conf'), 'cl_verbose_set', 'cl_dispatch_conf'),

Loading…
Cancel
Save