Browse Source

Добавлена поддержка binhost

tags/3.4.0_alpha1
Mike Khiretskiy 6 years ago
parent
commit
afdd27ff23
5 changed files with 254 additions and 80 deletions
  1. +0
    -1
      pym/update/emerge_parser.py
  2. +45
    -30
      pym/update/update.py
  3. +2
    -2
      pym/update/utils/cl_update.py
  4. +6
    -5
      pym/update/utils/cl_update_profile.py
  5. +201
    -42
      pym/update/variables/update.py

+ 0
- 1
pym/update/emerge_parser.py View File

@@ -94,7 +94,6 @@ class CommandExecutor(object):
if self.child:
self.child.send(s)


class ChrootCommandExecutor(CommandExecutor):
"""
Команда запускаемая в chroot


+ 45
- 30
pym/update/update.py View File

@@ -34,7 +34,6 @@ from calculate.update.update_info import UpdateInfo
from calculate.lib.cl_log import log
import re
from collections import MutableSet
from mock import self
from update_tasks import EmergeMark

from calculate.lib.utils.portage import (Git, Layman, EmergeLog, GitError,
@@ -206,11 +205,15 @@ class Update(object):
# git.resetRepository(rpath, to_rev=revision)
# info_outdate = True
if info_outdate:
self.raiseOutdate()
dv.Set('cl_update_outdate_set', 'on', force=True)
finally:
self.unstash_cache(rpath, name)
return True

def raiseOutdate(self):
self.clVars.Set('cl_update_outdate_set', 'on', force=True)

def setAutocheckParams(self, status, interval, update_other, cleanpkg):
"""
Настроить параметры автопроверки обновлений
@@ -305,6 +308,8 @@ class Update(object):
raise UpdateError(_("Configuration variables for repositories "
"are not setup"))
chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
# TODO: DEBUG
# print "DEBUG", repname, revision, branch
if chroot_path == '/':
rpath_orig = rpath
else:
@@ -417,6 +422,7 @@ class Update(object):
layman = self.get_prog_path('/usr/bin/layman')
if not layman:
raise UpdateError(_("The Layman tool is not found"))

rpath = self.clVars.Select('cl_update_other_rep_path',
where='cl_update_other_rep_name', eq=repname,
limit=1)
@@ -815,7 +821,7 @@ class Update(object):
"""
Установить флаг: есть обновления
"""
if self.clVars.Get('cl_update_autocheck_set') == 'off':
if self.clVars.Get('update.cl_update_autocheck_set') == 'off':
val = False
UpdateInfo.set_update_ready(val)
return True
@@ -1000,12 +1006,12 @@ class Update(object):
'cl_profile_system',
'cl_update_rep'):
# TODO: debug block
print var_name, ":", profile_dv.Get(var_name)
#print var_name, ":", profile_dv.Get(var_name)
dv.Set(var_name, profile_dv.Get(var_name), force=True)
dv.Set('cl_chroot_path', chroot, force=True)
# TODO: debug block
print ('cl_builder_branch_name',
self.clVars.Get('cl_builder_branch_name'))
#print ('cl_builder_branch_name',
# self.clVars.Get('cl_builder_branch_name'))
except DataVarsError as e:
raise UpdateError(_("Wrong profile"))
return True
@@ -1027,29 +1033,30 @@ class Update(object):
raise UpdateError(_("Failed to set the profile: %s")%str(e))
return True

def applyProfileTemplates(self,useClt=None,cltFilter=False,useDispatch=True):
def applyProfileTemplates(self, useClt=None, cltFilter=False,
useDispatch=True, action="merge"):
"""
Наложить шаблоны из профиля
"""
from calculate.lib.cl_template import (Template,TemplatesError,
ProgressTemplate)
from calculate.lib.cl_template import TemplatesError, ProgressTemplate
dv = DataVarsUpdate()
try:
dv.importUpdate()
dv.flIniFile()
dv.Set('cl_action','merge',force=True)
dv.Set('cl_templates_locate', self.clVars.Get('cl_update_templates_locate'))
dv.Set("cl_chroot_path",'/', True)
dv.Set("cl_root_path",'/', True)
for copyvar in ("cl_dispatch_conf", "cl_verbose_set"):
dv.Set(copyvar,self.clVars.Get(copyvar),True)
dv.Set('cl_action', action, force=True)
dv.Set('cl_templates_locate',
self.clVars.Get('cl_update_templates_locate'))
dv.Set("cl_chroot_path", '/', True)
dv.Set("cl_root_path", '/', True)
for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
"update.cl_update_world"):
dv.Set(copyvar, self.clVars.Get(copyvar), True)
# определение каталогов содержащих шаблоны
useClt = useClt in ("on",True)
useClt = useClt in ("on", True)
self.addProgress()
nullProgress = lambda *args,**kw:None
nullProgress = lambda *args, **kw: None
dispatch = self.dispatchConf if useDispatch else None
clTempl = ProgressTemplate(nullProgress, dv,
cltObj=useClt,
clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt,
cltFilter=cltFilter,
printSUCCESS=self.printSUCCESS,
printWARNING=self.printWARNING,
@@ -1071,9 +1078,6 @@ class Update(object):
return True

def cleanpkg(self):
"""
Очистить distfiles и pkgdir от устаревших пакетов
"""
portdirs = ([self.clVars.Get('cl_portdir')] +
self.clVars.Get('cl_portdir_overlay'))
pkgfiles = get_packages_files_directory(*portdirs)
@@ -1081,6 +1085,20 @@ class Update(object):
distdir = self.clVars.Get('install.cl_distfiles_path')
pkgdir = self.clVars.Get('cl_pkgdir')

logger = log("update_cleanpkg.log",
filename="/var/log/calculate/update_cleanpkg.log",
formatter="%(asctime)s - %(clean)s - %(message)s")

return self._cleanpkg(
distdir, pkgdir, distdirfiles, pkgfiles, logger)

def _update_binhost_packages(self):
os.system('/usr/sbin/emaint binhost -f &>/dev/null')

def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger):
"""
Очистить distfiles и pkgdir от устаревших пакетов
"""
skip_files = ["/metadata.dtd", "/Packages"]
try:
if self.clVars.Get('client.os_remote_auth'):
@@ -1088,14 +1106,11 @@ class Update(object):
except DataVarsError:
pass

logger = log("update_cleanpkg.log",
filename="/var/log/calculate/update_cleanpkg.log",
formatter="%(asctime)s - %(clean)s - %(message)s")

for cleantype, filelist in (("packages",
get_remove_list(pkgdir, list(pkgfiles), depth=4)),
("distfiles",
get_remove_list(distdir, list(distdirfiles), depth=1))):
for cleantype, filelist in (
("packages",
get_remove_list(pkgdir, list(pkgfiles), depth=4)),
("distfiles",
get_remove_list(distdir, list(distdirfiles), depth=1))):
removelist = []
for fn in filelist:
try:
@@ -1109,7 +1124,7 @@ class Update(object):
logger.info(removelist_str, extra={'clean': cleantype})
if cleantype == "packages":
try:
os.system('/usr/sbin/emaint binhost -f &>/dev/null')
self._update_binhost_packages()
for dn in listDirectory(pkgdir, fullPath=True):
if path.isdir(dn) and not listDirectory(dn):
os.rmdir(dn)


+ 2
- 2
pym/update/utils/cl_update.py View File

@@ -272,8 +272,8 @@ class ClUpdateAction(Action):
]
}
] + emerge_tasks + [
{'name':'failed',
'error':__("Update failed"),
{'name': 'failed',
'error': __("Update failed"),
'depend': (Tasks.failed() & Tasks.hasnot("interrupt") &
(Tasks.hasnot("check_schedule") |
Tasks.success_all("check_schedule")))},


+ 6
- 5
pym/update/utils/cl_update_profile.py View File

@@ -113,15 +113,16 @@ class ClUpdateProfileAction(Action):
},
{'name': 'revision',
'message': __("Fixing the settings"),
'method': 'Update.applyTemplates(install.cl_source,'
'cl_template_clt_set,True,None,False)',
'method': 'Update.applyProfileTemplates(cl_template_clt_set,'
'True,False,"update_profile")',
'condition': lambda Get: Get('cl_templates_locate')
},
{'name': 'reconfigure',
'message': __("The system is being configured"),
'method': 'Update.applyProfileTemplates(cl_template_clt_set,True,False)',
'condition': lambda Get: (Get('cl_update_templates_locate')
and Get('cl_update_skip_setup_set') == 'off')
'method': 'Update.applyProfileTemplates(cl_template_clt_set,'
'True,False,"merge")',
'condition': lambda Get: (Get('cl_update_templates_locate') and
Get('cl_update_skip_setup_set') == 'off')
},
{'name': 'dispatch_conf',
'message': __("Updating configuration files"),


+ 201
- 42
pym/update/variables/update.py View File

@@ -35,6 +35,8 @@ from ..profile import RepositoryStorageSet, DEFAULT_BRANCH, \
from calculate.lib.variables import linux as lib_linux
from calculate.lib.variables import env
from calculate.update.update_info import UpdateInfo
import urllib2
import time
from functools import partial

_ = lambda x:x
@@ -165,11 +167,13 @@ class VariableClUpdateRepPath(ReadonlyVariable):
Пути до репозиториев
"""
type = "list"
mapPath = {'portage':'usr/portage'}
mapPath = {'portage': 'usr/portage',
'gentoo': '/usr/portage'}

def get(self):
repPath = self.Get('cl_update_layman_storage')
chroot_path = self.Get('cl_chroot_path')

def generatePaths(names):
for name in names:
if name in self.mapPath:
@@ -217,8 +221,7 @@ class VariableClUpdateBranch(TableVariable):
self.help = _("set branches for repository (REPOSITORY:BRANCH)")
self.label = _("Repositories branches")

def raiseReadonlyIndexError(self,fieldname="",variablename="",
value=""):
def raiseReadonlyIndexError(self, fieldname="", variablename="", value=""):
"""
Неизвестный оврелей
"""
@@ -236,6 +239,94 @@ class VariableClUpdateBranchRep(ReadonlyVariable):
def get(self):
return self.Get('cl_update_rep_name')

class VariableClUpdateBinhostData(ReadonlyTableVariable):
"""
Содержимое файла revisions
"""
source = ["cl_update_binhost_host",
"cl_update_binhost_revisions",
"cl_update_binhost_time"]

def check_binhost(self, binhost):
revision_files = [path.join(binhost, x)
for x in self.Get('cl_update_revision_path')]
timeout = self.GetInteger('cl_update_binhost_timeout')
try:
data = None
t = time.time()
for fn in revision_files:
if data is None:
data = urllib2.urlopen(fn, timeout=timeout).read()
elif data != urllib2.urlopen(fn, timeout=timeout).read():
return "", str(-1)
return data, str(int((time.time() - t)*1000))
except urllib2.URLError:
return "", str(-1)

re_revison = re.compile("\w+=(\w+)")

def binhost_key(self, data):
host, data, time = data
try:
cp = ConfigParser()
cp.read_string(data.decode('utf-8'))
data = sum(int(x) for x in cp['vcs'].values())
except (TypeError, KeyError) as e:
data = 0
return (1 if int(time) >= 0 else 0,
data,
-int(time))

def get(self, hr=False):
def generate():
recheck = self.GetBool('cl_update_binhost_recheck_set')
base_dn = self.Get('cl_update_binhost')
for host in [base_dn] + [x for x in self.Get('cl_update_binhosts')
if x != base_dn]:
if host:
data, t = self.check_binhost(host)
yield host, data, str(t)
if not recheck and t >= 0 and base_dn == host:
break
return list(sorted(generate(), key=self.binhost_key, reverse=True))


class VariableClUpdateBinhostRecheckSet(Variable):
"""
Принудительно обновить binhost
"""
value = "off"

class VariableClUpdateBinhostHost(FieldValue,ReadonlyVariable):
"""
Список имен прочих репозиториев
"""
type = "list"
source_variable = "cl_update_binhost_data"
column = 0

class VariableClUpdateBinhostRevisions(FieldValue,ReadonlyVariable):
"""
Список имен прочих репозиториев
"""
type = "list"
source_variable = "cl_update_binhost_data"
column = 1

class VariableClUpdateBinhostTime(FieldValue,ReadonlyVariable):
"""
Список имен прочих репозиториев
"""
type = "list"
source_variable = "cl_update_binhost_data"
column = 2

class VariableClUpdateRepoTagSet(Variable):
"""
Флаг принудительного использования тэгов вместо веток
"""
type = "bool"
value = "on"

class VariableClUpdateBranchName(Variable):
"""
@@ -250,13 +341,24 @@ class VariableClUpdateBranchName(Variable):
return ["master", "develop", "update"]

def get(self):
cp = ConfigParser()
cp.read_string(unicode(
self.Get('update.cl_update_binhost_revisions')[0]))
repo_tag = self.GetBool('update.cl_update_repo_tag_set')

def generateBranch():
git = Git()
for reppath in self.Get('cl_update_rep_path'):
for reppath, repname in zip(self.Get('cl_update_rep_path'),
self.Get('cl_update_rep_name')):
tag = cp.get("vcs", repname, fallback="master")
try:
yield git.getBranch(reppath) or "master"
if (repo_tag or git.isTagRepository(
reppath) or not git.is_git(reppath)):
yield tag
else:
yield git.getBranch(reppath) or tag
except GitError:
yield "master"
yield tag

return list(generateBranch())

@@ -271,20 +373,24 @@ class VariableClUpdateSyncRep(Variable):
metavalue = "REPOSITORIES"
untrusted = True

@property
def rep_name(self):
return self.Get('update.cl_update_rep_name')

def init(self):
self.help = _("synchronized repositories (all by default)")
self.label = _("Synchronized repositories")

def set(self,value):
orderList = self.Get('cl_update_rep_name')
return sorted(value,key=lambda x:
(orderList.index(x) if x in orderList else -1),reverse=True)
def set(self, value):
orderList = self.rep_name
return sorted(value, key=lambda x:
(orderList.index(x) if x in orderList else -1), reverse=True)

def get(self):
return list(reversed(self.Get('cl_update_rep_name')))
return list(reversed(self.rep_name))

def choice(self):
return self.Get('cl_update_rep_name')
return self.rep_name

class VariableClUpdateSyncOverlayRep(ReadonlyVariable):
"""
@@ -293,7 +399,7 @@ class VariableClUpdateSyncOverlayRep(ReadonlyVariable):
type = "list"

def get(self):
return filter(lambda x:x!="portage",self.Get('cl_update_sync_rep'))
return filter(lambda x: x != "portage", self.Get('cl_update_sync_rep'))

class VariableClUpdateOutdateSet(ReadonlyVariable):
"""
@@ -399,16 +505,18 @@ class VariableClUpdateOtherRepData(ReadonlyTableVariable):
source = ['cl_update_other_rep_name',
'cl_update_other_rep_path']

portdir_overlay = "main.cl_portdir_overlay"

def generator(self):
repNames = self.Get('cl_update_rep_name')
layman = Layman(self.Get('cl_update_layman_installed'),
self.Get('cl_update_layman_make'),
self.Get('cl_update_layman_conf'))
repNames = self.Get('update.cl_update_rep_name')
layman = Layman(self.Get('update.cl_update_layman_installed'),
self.Get('update.cl_update_layman_make'),
self.Get('update.cl_update_layman_conf'))
layman_overlays = layman.get_installed()
for rpath in self.Get('cl_portdir_overlay'):
for rpath in self.Get(self.portdir_overlay):
repo_file = path.join(rpath, "profiles/repo_name")
rname = readFile(repo_file).strip() or path.basename(rpath)
if rname in layman_overlays and not rname in repNames:
if rname in layman_overlays and rname not in repNames:
yield (rname, rpath)

def get(self):
@@ -1043,31 +1151,45 @@ class DataVarsUpdateProfile(SimpleDataVars):
'cl_update_rep_path',
'cl_update_rep_rev',
'cl_update_branch_name']
def __init__(self, profile, chroot_path='/'):
SimpleDataVars.__init__(self,
lib_linux.VariableOsLinuxName(),
lib_linux.VariableOsLinuxShortname(),
lib_linux.VariableOsLinuxSubname(),
lib_linux.VariableOsLinuxVer(),
lib_linux.VariableClProfileSystem(),
env.VariableClRepositoryData(),
env.VariableClRepositoryName(),
env.VariableClRepositoryLocation(),
env.VariableClChrootPath(),
env.VariableClTemplateLocation(),
env.VariableClTemplatePath(),
env.VariableClEmergeConfig(systemRoot=chroot_path),
VariableClUpdateRepData(section="update"),
VariableClUpdateRepPath(section="update"),
VariableClUpdateRepRev(section="update"),
VariableClUpdateBranchName(section="update"),
VariableClUpdateLaymanConfig(section="update"),
VariableClUpdateLaymanStorage(section="update"),
VariableClUpdateRepName(section="update"),
VariableClUpdateRep(section="update"),
VariableClUpdateRepUrl(section="update"))
def __init__(self, profile, chroot_path='/', recheck=None):
SimpleDataVars.__init__(
self,
lib_linux.VariableOsLinuxName(),
lib_linux.VariableOsLinuxShortname(),
lib_linux.VariableOsLinuxSubname(),
lib_linux.VariableOsLinuxVer(),
lib_linux.VariableClProfileSystem(),
env.VariableClRepositoryData(),
env.VariableClRepositoryName(),
env.VariableClRepositoryLocation(),
env.VariableClChrootPath(),
env.VariableClTemplateLocation(),
env.VariableClTemplatePath(),
env.VariableClEmergeConfig(systemRoot=chroot_path),
VariableClUpdateRepData(section="update"),
VariableClUpdateRepPath(section="update"),
VariableClUpdateRepRev(section="update"),
VariableClUpdateBranchName(section="update"),
VariableClUpdateLaymanConfig(section="update"),
VariableClUpdateLaymanStorage(section="update"),
VariableClUpdateRepName(section="update"),
VariableClUpdateRep(section="update"),
VariableClUpdateRepUrl(section="update"),
VariableClUpdateBinhost(section="update"),
VariableClUpdateBinhostData(section="update"),
VariableClUpdateBinhostHost(section="update"),
VariableClUpdateBinhostRecheckSet(section="update"),
VariableClUpdateBinhostRevisions(section="update"),
VariableClUpdateBinhostTime(section="update"),
VariableClUpdateBinhostTimeout(section="update"),
VariableClUpdateBinhosts(section="update"),
VariableClUpdateRepoTagSet(section="update"),
VariableClUpdateRevisionPath(section="update"),
)
self.cache['cl_profile_system'] = profile
self.cache['cl_chroot_path'] = chroot_path
if recheck is not None:
self.cache['cl_update_binhost_recheck_set'] = recheck
self.flIniFileFrom(profile)

def __repr__(self):
@@ -1247,3 +1369,40 @@ class VariableClUpdateOutdatedKernelSet(ReadonlyVariable):
def get(self):
ui = UpdateInfo(self.parent)
return "on" if ui.outdated_kernel else "off"

class VariableClUpdateBinhosts(Variable):
"""
Список хостов с бинарными обновлениями
"""
type = "list"
value = ["ftp://ftp.calculate.ru/pub", 'ftp://localhost/pub']

class VariableClUpdateBinhost(Variable):
"""
Хост с бинарными обновлениями
"""
def get(self):
return ""
#binhosts = self.Get('cl_update_binhosts')
#if binhosts:
# return self.Get('cl_update_binhosts')[0]
#return ""

class VariableClUpdateRevisionPath(Variable):
"""
Путь до revisions файлов
"""
type = "list"
value = [
"calculate/grp/default/ini.env",
"calculate/grp/kde/ini.env",
"calculate/grp/server/ini.env",
"calculate/grp/x/ini.env"
]

class VariableClUpdateBinhostTimeout(Variable):
"""
Таймаут на проверку одного binhost
"""
type = "int"
value = "15"

Loading…
Cancel
Save