Изенена синхронизация до веток и тэгов

develop
Mike Khiretskiy 9 years ago
parent afdd27ff23
commit 8c39a51e32

@ -68,9 +68,13 @@ class CommandExecutor(object):
if logfile: if logfile:
self.logfile = logfile self.logfile = logfile
def get_command(self):
return [self.cmd] + self.params
def execute(self): def execute(self):
if self.child is None: if self.child is None:
self.child = pexpect.spawn(self.cmd, command_data = self.get_command()
self.child = pexpect.spawn(command_data[0], command_data[1:],
logfile=open(self.logfile, 'w'), logfile=open(self.logfile, 'w'),
env=self.env, cwd=self.cwd, timeout=None) env=self.env, cwd=self.cwd, timeout=None)
return self.child return self.child
@ -98,21 +102,15 @@ class ChrootCommandExecutor(CommandExecutor):
""" """
Команда запускаемая в chroot Команда запускаемая в chroot
""" """
def __init__(self, chroot_path, def __init__(self, chroot_path, cmd, params, env=None, cwd=None,
cmd, params, env=None, cwd=None, logfile=None): logfile=None):
self.chroot_path = chroot_path self.chroot_path = chroot_path
super(ChrootCommandExecutor, self).__init__(cmd, params, env=env, super(ChrootCommandExecutor, self).__init__(cmd, params, env=env,
cwd=cwd, logfile=logfile) cwd=cwd, logfile=logfile)
def execute(self): def get_command(self):
if self.child is None: chrootCmd = '/usr/bin/chroot'
chrootCmd = '/usr/bin/chroot' return [chrootCmd, self.chroot_path, self.cmd] + self.params
self.child = pexpect.spawn(chrootCmd, [self.chroot_path, self.cmd] +
list(self.params),
logfile=open(self.logfile, 'w'),
env=self.env, cwd=self.cwd, timeout=None)
return self.child
class EmergeCommand(CommandExecutor): class EmergeCommand(CommandExecutor):
@ -146,12 +144,8 @@ class EmergeCommand(CommandExecutor):
if logfile: if logfile:
self.logfile = logfile self.logfile = logfile
def execute(self): def get_command(self):
if self.child is None: return [self.cmd] + self.params + self.packages
self.child = pexpect.spawn(self.cmd, self.params + self.packages,
logfile=open(self.logfile, 'w'),
env=self.env, cwd=self.cwd, timeout=None)
return self.child
class ChrootEmergeCommand(EmergeCommand): class ChrootEmergeCommand(EmergeCommand):
@ -160,26 +154,53 @@ class ChrootEmergeCommand(EmergeCommand):
super(ChrootEmergeCommand, self).__init__(*args, **kw) super(ChrootEmergeCommand, self).__init__(*args, **kw)
def execute(self): def get_command(self):
if self.child is None: chrootCmd = '/usr/bin/chroot'
chrootCmd = '/usr/bin/chroot' bashCmd = '/bin/bash'
bashCmd = '/bin/bash' bash_command = (
bash_command = ( "env-update &>/dev/null;"
"env-update &>/dev/null;" "source /etc/profile &>/dev/null;"
"source /etc/profile &>/dev/null;" "{emerge_command} {params} {packages}".format(
"{emerge_command} {params} {packages}".format( emerge_command=self.cmd,
emerge_command=self.cmd, params=" ".join(self.params),
params=" ".join(self.params), packages=" ".join(self.packages)
packages=" ".join(self.packages) ))
)) return [chrootCmd, self.chroot_path, bashCmd, "-c",
# TODO: использование linux32 bash_command]
self.child = pexpect.spawn(chrootCmd,
[self.chroot_path, bashCmd, def Chroot(chroot_path, obj):
"-c", bash_command], """
logfile=open(self.logfile, 'w'), Преобразовать команду (экземпляр объекта) в chroot
env=self.env, cwd=self.cwd, :param obj: экземпляр команды
timeout=None) :param chroot_path: путь для chroot
return self.child :return:
"""
old_get_command = obj.get_command
def get_command():
chrootCmd = '/usr/bin/chroot'
bashCmd = '/bin/bash'
bash_command = (
"env-update &>/dev/null;"
"source /etc/profile &>/dev/null;"
"{cmd}".format(cmd=" ".join(old_get_command())))
return [chrootCmd, chroot_path, bashCmd, "-c", bash_command]
obj.get_command = get_command
return obj
def Linux32(obj):
"""
Преобразовать команду (экземпляр объекта) в вызов под linux32
:param obj: экземпляр команды
:return:
"""
old_get_command = obj.get_command
def get_command():
return ["/usr/bin/linux32"] + old_get_command()
obj.get_command = get_command
return obj
class EmergeInformationBlock(object): class EmergeInformationBlock(object):

@ -192,9 +192,10 @@ class ProfileRepository(object):
repo_name = pr.repo_name repo_name = pr.repo_name
if name != repo_name: if name != repo_name:
rpath_new = path.join(storage.directory, repo_name) rpath_new = path.join(storage.directory, repo_name)
if not path.exists(rpath_new): if path.exists(rpath_new):
os.rename(rpath, rpath_new) removeDir(rpath_new)
pr = cls(repo_name, storage) os.rename(rpath, rpath_new)
pr = cls(repo_name, storage)
return pr return pr
@property @property

@ -140,7 +140,7 @@ class Update(object):
def get_prog_path(self, progname): def get_prog_path(self, progname):
return getProgPath(progname) return getProgPath(progname)
def _syncRepository(self, name, url, rpath, revision, branch, def _syncRepository(self, name, url, rpath, revision,
cb_progress=None): cb_progress=None):
""" """
Синхронизировать репозитори Синхронизировать репозитори
@ -151,59 +151,26 @@ class Update(object):
try: try:
self.stash_cache(rpath, name) self.stash_cache(rpath, name)
if not git.checkExistsRep(rpath): if not git.checkExistsRep(rpath):
if revision == "last": git.cloneTagRepository(url, rpath, revision,
git.cloneTagRepository(url, rpath, branch, cb_progress=cb_progress)
cb_progress=cb_progress)
else:
git.cloneTagRepository(url, rpath, revision,
cb_progress=cb_progress)
info_outdate = True info_outdate = True
else: else:
if revision != "last": try:
try: cr = ""
need_update = False need_update = False
tag_cr = git.getCommit(rpath, revision) tag_cr = git.getCommit(rpath, revision)
cr = git.getCurrentCommit(rpath) cr = git.getCurrentCommit(rpath)
if tag_cr != cr: ref_type = git.reference_type(rpath, revision)
need_update = True if tag_cr != cr or ref_type == Git.Reference.Branch:
except GitError:
need_update = True need_update = True
if need_update: except GitError:
git.updateTagRepository(url, rpath, revision, need_update = True
cb_progress=cb_progress) if need_update:
info_outdate = True git.updateTagRepository(url, rpath, revision,
else:
try:
old_cr = git.getCurrentCommit(rpath)
except GitError:
old_cr = ""
git.updateTagRepository(url, rpath, branch,
cb_progress=cb_progress) cb_progress=cb_progress)
if old_cr != git.getCurrentCommit(rpath): new_cr = git.getCurrentCommit(rpath)
if new_cr != cr:
info_outdate = True info_outdate = True
# если нужно обновиться до конкретной ревизии
#if revision != "last":
# if revision == git.getCurrentCommit(rpath):
# if git.getBranch(rpath) == branch:
# return False
## получить изменения из удаленного репозитория
#git.fetchRepository(rpath, cb_progress=cb_progress)
## если текущая ветка не соответствует нужной
#repInfo = git.getStatusInfo(rpath)
#if repInfo['branch'] != branch:
# # меняем ветку
# info_outdate = True
# git.checkoutBranch(rpath, branch)
#if revision == "last":
# if git.resetRepository(rpath, to_origin=True):
# # если не удалось сбросить
# repInfo = git.getStatusInfo(rpath)
# if repInfo.get("files", False):
# raise GitError("Failed to reset git")
# info_outdate = True
#else:
# git.resetRepository(rpath, to_rev=revision)
# info_outdate = True
if info_outdate: if info_outdate:
self.raiseOutdate() self.raiseOutdate()
dv.Set('cl_update_outdate_set', 'on', force=True) dv.Set('cl_update_outdate_set', 'on', force=True)
@ -300,21 +267,22 @@ class Update(object):
Синхронизировать репозитории Синхронизировать репозитории
""" """
dv = self.clVars dv = self.clVars
url, rpath, revision, branch = ( url, rpath, revision = (
dv.Select(["cl_update_rep_url", "cl_update_rep_path", dv.Select(["cl_update_rep_url", "cl_update_rep_path",
"cl_update_rep_rev", "cl_update_branch_name"], "cl_update_rep_rev"],
where="cl_update_rep_name", eq=repname, limit=1)) where="cl_update_rep_name", eq=repname, limit=1))
if not url or not rpath: if not url or not rpath:
raise UpdateError(_("Configuration variables for repositories " raise UpdateError(_("Configuration variables for repositories "
"are not setup")) "are not setup"))
chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
# TODO: DEBUG # TODO: DEBUG
# print "DEBUG", repname, revision, branch print "DEBUG", repname, revision
if chroot_path == '/': if chroot_path == '/':
rpath_orig = rpath rpath_orig = rpath
else: else:
rpath_orig = rpath[len(chroot_path):] rpath_orig = rpath[len(chroot_path):]
self.addProgress() self.addProgress()
if clean_on_error: if clean_on_error:
try: try:
layman = Layman(dv.Get('cl_update_layman_installed'), layman = Layman(dv.Get('cl_update_layman_installed'),
@ -322,7 +290,7 @@ class Update(object):
dv.Get('cl_update_layman_conf')) dv.Get('cl_update_layman_conf'))
if repname != "portage": if repname != "portage":
layman.add(repname, url, rpath_orig) layman.add(repname, url, rpath_orig)
if not self._syncRepository(repname, url, rpath, revision, branch, if not self._syncRepository(repname, url, rpath, revision,
cb_progress=self.setProgress): cb_progress=self.setProgress):
return "skip" return "skip"
return True return True
@ -337,7 +305,7 @@ class Update(object):
rpath_new = "%s_new" % rpath rpath_new = "%s_new" % rpath
try: try:
self._syncRepository(repname, url, rpath_new, revision, self._syncRepository(repname, url, rpath_new, revision,
branch, cb_progress=self.setProgress) cb_progress=self.setProgress)
removeDir(rpath) removeDir(rpath)
os.rename(rpath_new, rpath) os.rename(rpath_new, rpath)
except OSError as e: except OSError as e:
@ -998,13 +966,16 @@ class Update(object):
if not profile_dv: if not profile_dv:
raise UpdateError( raise UpdateError(
_("Failed to use the new profile. Try again.")) _("Failed to use the new profile. Try again."))
for var_name in ('cl_update_rep_rev', for var_name in (#'cl_update_rep_rev',
'cl_update_rep_path', 'cl_update_rep_path',
'cl_update_rep_url', 'cl_update_rep_url',
'cl_update_rep_name', 'cl_update_rep_name',
'cl_update_branch',
'cl_update_binhosts',
'cl_update_branch_name', 'cl_update_branch_name',
'cl_profile_system', 'cl_profile_system',
'cl_update_rep'): 'cl_update_rep'
):
# TODO: debug block # TODO: debug block
#print var_name, ":", profile_dv.Get(var_name) #print var_name, ":", profile_dv.Get(var_name)
dv.Set(var_name, profile_dv.Get(var_name), force=True) dv.Set(var_name, profile_dv.Get(var_name), force=True)
@ -1013,6 +984,7 @@ class Update(object):
#print ('cl_builder_branch_name', #print ('cl_builder_branch_name',
# self.clVars.Get('cl_builder_branch_name')) # self.clVars.Get('cl_builder_branch_name'))
except DataVarsError as e: except DataVarsError as e:
print str(e)
raise UpdateError(_("Wrong profile")) raise UpdateError(_("Wrong profile"))
return True return True
@ -1025,11 +997,16 @@ class Update(object):
self.clVars.Get('cl_update_profile_system')) self.clVars.Get('cl_update_profile_system'))
profile_path = path.relpath(profile, '/etc/portage') profile_path = path.relpath(profile, '/etc/portage')
try: try:
for rm_fn in filter(path.exists, profile_file = '/etc/portage/make.profile'
if not path.exists(
path.join(path.dirname(profile_file), profile_path)):
raise UpdateError(
_("Failed to set the profile: %s")%_("Profile not found"))
for rm_fn in filter(path.lexists,
('/etc/make.profile', '/etc/portage/make.profile')): ('/etc/make.profile', '/etc/portage/make.profile')):
os.unlink(rm_fn) os.unlink(rm_fn)
os.symlink(profile_path, '/etc/portage/make.profile') os.symlink(profile_path, profile_file)
except (OSError,IOError) as e: except (OSError, IOError) as e:
raise UpdateError(_("Failed to set the profile: %s")%str(e)) raise UpdateError(_("Failed to set the profile: %s")%str(e))
return True return True
@ -1078,6 +1055,9 @@ class Update(object):
return True return True
def cleanpkg(self): def cleanpkg(self):
"""
Очистить PKGDIR и DISTFILES в текущей системе
"""
portdirs = ([self.clVars.Get('cl_portdir')] + portdirs = ([self.clVars.Get('cl_portdir')] +
self.clVars.Get('cl_portdir_overlay')) self.clVars.Get('cl_portdir_overlay'))
pkgfiles = get_packages_files_directory(*portdirs) pkgfiles = get_packages_files_directory(*portdirs)
@ -1097,7 +1077,7 @@ class Update(object):
def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger): def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger):
""" """
Очистить distfiles и pkgdir от устаревших пакетов Общий алгоритм очистки distfiles и pkgdir от устаревших пакетов
""" """
skip_files = ["/metadata.dtd", "/Packages"] skip_files = ["/metadata.dtd", "/Packages"]
try: try:
@ -1136,3 +1116,52 @@ class Update(object):
cache = SetupCache(self.clVars) cache = SetupCache(self.clVars)
cache.update(force=True) cache.update(force=True)
return True return True
def check_binhost(self, write_binhost=True):
"""
Проверить, что доступен хотя бы один из binhost'ов
:return:
"""
hosts =self.clVars.Get("update.cl_update_binhost_host")
if not hosts:
self.clVars.Delete('cl_update_binhost', location="system")
raise UpdateError("Binhost is unavailable")
if write_binhost:
self.clVars.Write('cl_update_binhost', hosts[0], location="system")
return True
def update_binhost_list(self):
"""
Обновить список binhost'ов после обновления до master веток
:return:
"""
dv = DataVarsUpdate()
try:
dv.importUpdate()
dv.flIniFile()
changes = False
for varname in ('update.cl_update_binhosts',
'update.cl_update_binhost_timestamp_path',
'cl_update_binhost_revision_path'):
new_value = dv.Get(varname)
old_value = self.clVars.Get(varname)
if new_value != old_value:
changes = True
self.clVars.Set(varname, new_value, force=True)
if not changes:
raise UpdateError("Binhost is unavailable")
except DataVarsError:
raise UpdateError("Binhost is unavailable")
return True
def drop_binhosts(self, dv):
"""
Обновление до master веток
"""
branch = dv.Get('update.cl_update_branch')
revs = [
branch for x in dv.Get('update.cl_update_rep_name')
]
dv.Set('update.cl_update_branch_name', revs)
dv.Invalidate('update.cl_update_rep_rev')
return True

@ -189,12 +189,32 @@ class ClUpdateAction(Action):
{'name': 'reps_synchronization', {'name': 'reps_synchronization',
'group': __("Repositories synchronization"), 'group': __("Repositories synchronization"),
'tasks': [ 'tasks': [
# запасная синхронизация, в ходе которой ветки обновляются до
# master
{'name': 'sync_reps_fallback',
'foreach': 'cl_update_sync_rep',
'message':
__("Fallback syncing the {eachvar:capitalize} repository"),
'method': 'Update.syncRepositories(eachvar)',
'condition': lambda Get: ("getbinpkg" in Get('cl_features') and
not Get('cl_update_binhost_data')[0])
},
# обновление переменных информации из binhost
{'name': 'update_binhost_list',
'method': 'Update.update_binhost_list()',
'condition': lambda Get: ("getbinpkg" in Get('cl_features') and
not Get('cl_update_binhost_data')[0])
},
{'name': 'sync_reps', {'name': 'sync_reps',
'foreach': 'cl_update_sync_rep', 'foreach': 'cl_update_sync_rep',
'message': __("Syncing the {eachvar:capitalize} repository"), 'message': __("Syncing the {eachvar:capitalize} repository"),
'method': 'Update.syncRepositories(eachvar)', 'method': 'Update.syncRepositories(eachvar)',
'condition': lambda Get: Get('cl_update_sync_rep') 'condition': lambda Get: Get('cl_update_sync_rep')
}, },
{'name': 'check_binhost',
'method': 'Update.check_binhost()',
'condition': lambda Get: "getbinpkg" in Get('cl_features')
},
{'name': 'sync_other_reps', {'name': 'sync_other_reps',
'foreach': 'cl_update_other_rep_name', 'foreach': 'cl_update_other_rep_name',
'message': __("Syncing the {eachvar:capitalize} repository"), 'message': __("Syncing the {eachvar:capitalize} repository"),

@ -53,6 +53,9 @@ class ClUpdateProfileAction(Action):
'method': 'Update.invalidateVariables("cl_update_profile_storage")', 'method': 'Update.invalidateVariables("cl_update_profile_storage")',
'depend': Tasks.has('migrate_repository') 'depend': Tasks.has('migrate_repository')
}, },
{'name': 'drop_binhosts',
'method': 'Update.drop_binhosts(update.cl_update_profile_datavars)'
},
{'name': 'reconfigure_vars', {'name': 'reconfigure_vars',
'method': 'Update.reconfigureProfileVars(cl_update_profile_datavars,' 'method': 'Update.reconfigureProfileVars(cl_update_profile_datavars,'
'cl_chroot_path)' 'cl_chroot_path)'
@ -86,8 +89,6 @@ class ClUpdateProfileAction(Action):
}, },
{'name': 'eix_update', {'name': 'eix_update',
'message': __("Updating the eix cache"), 'message': __("Updating the eix cache"),
# TODO: необходимо добавить опцию --changes-pkg emerge world
# TODO: возможно стоит использовать переменные от нового профиля
'method': 'Update.eixUpdate(cl_repository_name)', 'method': 'Update.eixUpdate(cl_repository_name)',
'condition': ( 'condition': (
lambda Get: (Get('cl_update_outdate_set') == 'on' and lambda Get: (Get('cl_update_outdate_set') == 'on' and

@ -21,8 +21,7 @@ from os import path
from calculate.lib.datavars import (Variable, VariableError, from calculate.lib.datavars import (Variable, VariableError,
ReadonlyVariable, ReadonlyTableVariable, TableVariable, FieldValue, ReadonlyVariable, ReadonlyTableVariable, TableVariable, FieldValue,
SimpleDataVars, DataVarsError) SimpleDataVars, DataVarsError)
from calculate.lib.utils.portage import searchProfile from calculate.lib.utils.files import readFile, \
from calculate.lib.utils.files import readLinesFile, readFile, makeDirectory, \
listDirectory, process, pathJoin listDirectory, process, pathJoin
from calculate.lib.configparser import ConfigParser from calculate.lib.configparser import ConfigParser
@ -37,7 +36,6 @@ from calculate.lib.variables import env
from calculate.update.update_info import UpdateInfo from calculate.update.update_info import UpdateInfo
import urllib2 import urllib2
import time import time
from functools import partial
_ = lambda x:x _ = lambda x:x
@ -121,8 +119,7 @@ class VariableClUpdateRepData(ReadonlyTableVariable):
source = ['cl_update_rep_name', source = ['cl_update_rep_name',
'cl_update_rep_url', 'cl_update_rep_url',
'cl_update_rep_path', 'cl_update_rep_path',
'cl_update_rep_rev', 'cl_update_rep_rev']
'cl_update_branch_name']
class VariableClUpdateRepName(Variable): class VariableClUpdateRepName(Variable):
""" """
@ -182,6 +179,7 @@ class VariableClUpdateRepPath(ReadonlyVariable):
yield path.join(repPath, name) yield path.join(repPath, name)
return list(generatePaths(self.Get('cl_update_rep_name'))) return list(generatePaths(self.Get('cl_update_rep_name')))
class VariableClUpdateRepRev(Variable): class VariableClUpdateRepRev(Variable):
""" """
Ревизии до которых необходимо обновить репозитории Ревизии до которых необходимо обновить репозитории
@ -189,25 +187,31 @@ class VariableClUpdateRepRev(Variable):
type = "list" type = "list"
def get(self): def get(self):
""" cp = ConfigParser()
Если cl_update_rep=rev, то профиль просматривается на нахождене revisions = self.Get('update.cl_update_binhost_revisions')
в нем rev файла (используется более частный вариант, if revisions:
как с calculate.env), файл содержит строки ключ=значение, где cp.read_string(unicode(revisions[0]))
ключ - название репозитория, значение - ревизия до которой нужно
обновиться branch = self.Get('cl_update_branch')
"""
if self.Get('cl_update_rep') == 'rev': def generateBranch():
revPaths = searchProfile(self.Get('cl_profile_system'), for repname, branchname in zip(
"rev") self.Get('cl_update_rep_name'),
if revPaths: self.Get('cl_update_branch_name')):
revPath = revPaths[-1] if branchname == "binhost":
dictNamesRevs = dict(map(lambda x:x.strip().partition('=')[::2], yield cp.get("vcs", repname, fallback=branch)
readLinesFile(revPath))) else:
return map(lambda x:dictNamesRevs.get(x,"last"), yield branchname
self.Get('cl_update_rep_name')) return list(generateBranch())
return ["last"]*len(self.Get('cl_update_rep_name'))
class VariableClUpdateBranch(TableVariable): class VariableClUpdateBranch(Variable):
"""
Ветка на которую будет обновляться репозиторий если -getbinpkg
"""
value = "master"
class VariableClUpdateBranchData(TableVariable):
""" """
Выбор веток репозиториев до которых необходимо обновиться Выбор веток репозиториев до которых необходимо обновиться
""" """
@ -241,7 +245,8 @@ class VariableClUpdateBranchRep(ReadonlyVariable):
class VariableClUpdateBinhostData(ReadonlyTableVariable): class VariableClUpdateBinhostData(ReadonlyTableVariable):
""" """
Содержимое файла revisions Таблица содержащая
binhost/содержимое файла ревизий/время доступа
""" """
source = ["cl_update_binhost_host", source = ["cl_update_binhost_host",
"cl_update_binhost_revisions", "cl_update_binhost_revisions",
@ -249,24 +254,23 @@ class VariableClUpdateBinhostData(ReadonlyTableVariable):
def check_binhost(self, binhost): def check_binhost(self, binhost):
revision_files = [path.join(binhost, x) revision_files = [path.join(binhost, x)
for x in self.Get('cl_update_revision_path')] for x in self.Get('cl_update_binhost_revision_path')]
timeout = self.GetInteger('cl_update_binhost_timeout') timeout = self.GetInteger('cl_update_binhost_timeout')
try: try:
data = None data = None
t = time.time()
for fn in revision_files: for fn in revision_files:
if data is None: if data is None:
data = urllib2.urlopen(fn, timeout=timeout).read() data = urllib2.urlopen(fn, timeout=timeout).read()
elif data != urllib2.urlopen(fn, timeout=timeout).read(): elif data != urllib2.urlopen(fn, timeout=timeout).read():
return "", str(-1) return ""
return data, str(int((time.time() - t)*1000)) return data
except urllib2.URLError: except urllib2.URLError:
return "", str(-1) return ""
re_revison = re.compile("\w+=(\w+)") re_revison = re.compile("\w+=(\w+)")
def binhost_key(self, data): def binhost_key(self, data):
host, data, time = data host, t = data
try: try:
cp = ConfigParser() cp = ConfigParser()
cp.read_string(data.decode('utf-8')) cp.read_string(data.decode('utf-8'))
@ -277,27 +281,59 @@ class VariableClUpdateBinhostData(ReadonlyTableVariable):
data, data,
-int(time)) -int(time))
def get_timestamp(self, binhost):
DAY = 60 * 60 * 24
timeout = self.GetInteger('cl_update_binhost_timeout')
timestamp_file = path.join(binhost,
self.Get('cl_update_binhost_timestamp_path'))
try:
t = time.time()
data = urllib2.urlopen(timestamp_file, timeout=timeout).read().strip()
if data.isdigit() and t - int(data) < 5 * DAY:
return data, int((time.time() - t)*1000)
except urllib2.URLError as e:
pass
return "", -1
def get(self, hr=False): def get(self, hr=False):
def generate(): binhost = self.Get('cl_update_binhost')
recheck = self.GetBool('cl_update_binhost_recheck_set') recheck = self.GetBool('cl_update_binhost_recheck_set')
base_dn = self.Get('cl_update_binhost')
for host in [base_dn] + [x for x in self.Get('cl_update_binhosts') def generate_by_timestamp():
if x != base_dn]: for host in self.Get('cl_update_binhosts'):
if host: if host:
data, t = self.check_binhost(host) ts_content, duration = self.get_timestamp(host)
yield host, data, str(t) if ts_content:
if not recheck and t >= 0 and base_dn == host: yield host, ts_content, str(duration)
break if not recheck and binhost:
return list(sorted(generate(), key=self.binhost_key, reverse=True)) ts, t = self.get_timestamp(binhost)
if ts:
data = self.check_binhost(binhost)
if data:
return [[binhost, data, str(t)]]
for host, ts, t in sorted(generate_by_timestamp(),
key=lambda x: (-int(x[1]), int(x[2]))):
data = self.check_binhost(host)
if data:
return [[host, data, str(t)]]
return [[]]
class VariableClUpdateBinhostRecheckSet(Variable): class VariableClUpdateBinhostRecheckSet(Variable):
""" """
Принудительно обновить binhost Принудительно обновить binhost
""" """
type = "bool"
value = "off" value = "off"
class VariableClUpdateBinhostHost(FieldValue,ReadonlyVariable): opt = ["--recheck-binhost"]
def init(self):
self.help = _("re-check binary hosts")
self.label = _("Re-check binary hosts")
class VariableClUpdateBinhostHost(FieldValue, ReadonlyVariable):
""" """
Список имен прочих репозиториев Список имен прочих репозиториев
""" """
@ -321,12 +357,6 @@ class VariableClUpdateBinhostTime(FieldValue,ReadonlyVariable):
source_variable = "cl_update_binhost_data" source_variable = "cl_update_binhost_data"
column = 2 column = 2
class VariableClUpdateRepoTagSet(Variable):
"""
Флаг принудительного использования тэгов вместо веток
"""
type = "bool"
value = "on"
class VariableClUpdateBranchName(Variable): class VariableClUpdateBranchName(Variable):
""" """
@ -338,29 +368,14 @@ class VariableClUpdateBranchName(Variable):
self.label = _("Branches") self.label = _("Branches")
def choice(self): def choice(self):
return ["master", "develop", "update"] return ["master", "develop", "update", "binhost"]
def get(self): def get(self):
cp = ConfigParser() if "getbinpkg" in self.Get('cl_features'):
cp.read_string(unicode( return ["binhost" for x in self.Get('cl_update_rep_name')]
self.Get('update.cl_update_binhost_revisions')[0])) else:
repo_tag = self.GetBool('update.cl_update_repo_tag_set') branch = self.Get('cl_update_branch')
return [branch for x in self.Get('cl_update_rep_name')]
def generateBranch():
git = Git()
for reppath, repname in zip(self.Get('cl_update_rep_path'),
self.Get('cl_update_rep_name')):
tag = cp.get("vcs", repname, fallback="master")
try:
if (repo_tag or git.isTagRepository(
reppath) or not git.is_git(reppath)):
yield tag
else:
yield git.getBranch(reppath) or tag
except GitError:
yield tag
return list(generateBranch())
class VariableClUpdateSyncRep(Variable): class VariableClUpdateSyncRep(Variable):
@ -657,7 +672,7 @@ class VariableClUpdateProfileLinuxFullname(ReadonlyVariable):
if subname: if subname:
return "%s %s %s" % (linuxname, linuxver, subname) return "%s %s %s" % (linuxname, linuxver, subname)
return "%s %s" %(linuxname,linuxver) return "%s %s" %(linuxname,linuxver)
except DataVarsError: except DataVarsError as s:
raise VariableError("Wrong Calculate Linux profile") raise VariableError("Wrong Calculate Linux profile")
return "" return ""
@ -914,6 +929,7 @@ class VariableClUpdateProfileBranch(Variable):
storage = "cl_update_profile_storage" storage = "cl_update_profile_storage"
url = "cl_update_profile_url" url = "cl_update_profile_url"
value = DEFAULT_BRANCH
def init(self): def init(self):
self.label = _("Repository branch") self.label = _("Repository branch")
@ -928,16 +944,6 @@ class VariableClUpdateProfileBranch(Variable):
#except GitError as e: #except GitError as e:
# raise VariableError(str(e)) # raise VariableError(str(e))
def get(self):
rep_set = self.Get(self.storage)
url = self.Get(self.url)
#print url, rep_set.is_local(url, branch=None)
if rep_set.is_local(url, branch=None):
rep = rep_set.get_repository(url, branch=None)
git = Git()
return git.getBranch(rep.directory)
return DEFAULT_BRANCH
class VariableClProfileRepository(ReadonlyVariable): class VariableClProfileRepository(ReadonlyVariable):
""" """
@ -1014,6 +1020,16 @@ class VariableClUpdateProfileRepository(ReadonlyVariable):
except GitError: except GitError:
return "" return ""
class VariableClUpdateProfileRepositoryName(ReadonlyVariable):
"""
Название репозитория, из которого будут извлечены профили
"""
def get(self):
rep = self.Get('cl_update_profile_repository')
if rep:
return rep.repo_name
return ""
class VariableClUpdateProfileData(VariableClProfileData): class VariableClUpdateProfileData(VariableClProfileData):
source = ["cl_update_profile_fullname", source = ["cl_update_profile_fullname",
@ -1166,9 +1182,11 @@ class DataVarsUpdateProfile(SimpleDataVars):
env.VariableClTemplateLocation(), env.VariableClTemplateLocation(),
env.VariableClTemplatePath(), env.VariableClTemplatePath(),
env.VariableClEmergeConfig(systemRoot=chroot_path), env.VariableClEmergeConfig(systemRoot=chroot_path),
env.VariableClFeatures(),
VariableClUpdateRepData(section="update"), VariableClUpdateRepData(section="update"),
VariableClUpdateRepPath(section="update"), VariableClUpdateRepPath(section="update"),
VariableClUpdateRepRev(section="update"), VariableClUpdateRepRev(section="update"),
VariableClUpdateBranch(section="update"),
VariableClUpdateBranchName(section="update"), VariableClUpdateBranchName(section="update"),
VariableClUpdateLaymanConfig(section="update"), VariableClUpdateLaymanConfig(section="update"),
VariableClUpdateLaymanStorage(section="update"), VariableClUpdateLaymanStorage(section="update"),
@ -1182,10 +1200,11 @@ class DataVarsUpdateProfile(SimpleDataVars):
VariableClUpdateBinhostRevisions(section="update"), VariableClUpdateBinhostRevisions(section="update"),
VariableClUpdateBinhostTime(section="update"), VariableClUpdateBinhostTime(section="update"),
VariableClUpdateBinhostTimeout(section="update"), VariableClUpdateBinhostTimeout(section="update"),
VariableClUpdateBinhostTimestampPath(section="update"),
VariableClUpdateBinhosts(section="update"), VariableClUpdateBinhosts(section="update"),
VariableClUpdateRepoTagSet(section="update"), VariableClUpdateBinhostRevisionPath(section="update"),
VariableClUpdateRevisionPath(section="update"),
) )
# TODO: при переключении профиля использовать master ветки
self.cache['cl_profile_system'] = profile self.cache['cl_profile_system'] = profile
self.cache['cl_chroot_path'] = chroot_path self.cache['cl_chroot_path'] = chroot_path
if recheck is not None: if recheck is not None:
@ -1375,20 +1394,15 @@ class VariableClUpdateBinhosts(Variable):
Список хостов с бинарными обновлениями Список хостов с бинарными обновлениями
""" """
type = "list" type = "list"
value = ["ftp://ftp.calculate.ru/pub", 'ftp://localhost/pub'] value = ["ftp://ftp.calculate.ru/pub"]
class VariableClUpdateBinhost(Variable): class VariableClUpdateBinhost(Variable):
""" """
Хост с бинарными обновлениями Хост с бинарными обновлениями
""" """
def get(self): value = ""
return ""
#binhosts = self.Get('cl_update_binhosts')
#if binhosts:
# return self.Get('cl_update_binhosts')[0]
#return ""
class VariableClUpdateRevisionPath(Variable): class VariableClUpdateBinhostRevisionPath(Variable):
""" """
Путь до revisions файлов Путь до revisions файлов
""" """
@ -1400,9 +1414,15 @@ class VariableClUpdateRevisionPath(Variable):
"calculate/grp/x/ini.env" "calculate/grp/x/ini.env"
] ]
class VariableClUpdateBinhostTimestampPath(Variable):
"""
Путь до файла timestamp
"""
value = "calculate/grp/timestamp"
class VariableClUpdateBinhostTimeout(Variable): class VariableClUpdateBinhostTimeout(Variable):
""" """
Таймаут на проверку одного binhost Таймаут на проверку одного binhost
""" """
type = "int" type = "int"
value = "15" value = "5"

@ -76,8 +76,9 @@ class Wsdl(WsdlBase):
'cl_update_eixupdate_force', 'cl_update_eixupdate_force',
'cl_update_skip_rb_set', 'cl_update_skip_rb_set',
'cl_update_wait_another_set', 'cl_update_wait_another_set',
'cl_update_branch', 'cl_update_branch_data',
'cl_update_autocheck_schedule_set', 'cl_update_autocheck_schedule_set',
'cl_update_binhost_recheck_set',
'cl_templates_locate', 'cl_templates_locate',
'cl_verbose_set', 'cl_dispatch_conf'), 'cl_verbose_set', 'cl_dispatch_conf'),
next_label=_("Perform"))]}, next_label=_("Perform"))]},
@ -116,8 +117,7 @@ class Wsdl(WsdlBase):
hide=('cl_update_profile_url', hide=('cl_update_profile_url',
'cl_update_profile_sync_set'), 'cl_update_profile_sync_set'),
normal=('cl_update_profile_url',), normal=('cl_update_profile_url',),
expert=('cl_update_profile_sync_set', expert=('cl_update_profile_sync_set',)),
'cl_update_profile_branch')),
lambda group: group(_("Profile"), lambda group: group(_("Profile"),
normal=('cl_update_profile_system', normal=('cl_update_profile_system',
'cl_update_world'), 'cl_update_world'),

Loading…
Cancel
Save