calculate
/
calculate-utils-3-update
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Улучшено обновление дистрибутива

Browse Source
Mike Khiretskiy 9 years ago
parent 08bbdc6142
commit 1dbc72df26
10 changed files with 526 additions and 139 deletions
Show Stats Download Patch File Download Diff File

175
pym/builder/builder.py
Unescape View File

@ -24,10 +24,14 @@ from calculate.lib.utils.files import (
makeDirectory, writeFile, readLinesFile, chmod, chown, FilePermission, makeDirectory, writeFile, readLinesFile, chmod, chown, FilePermission,
find, FindFileType, removeFileWithEmptyDirectory, find, FindFileType, removeFileWithEmptyDirectory,
copyWithPath) copyWithPath)
from calculate.lib.utils.git import Git
from calculate.lib.utils.portage import Layman, EmergeLog, EmergeLogNamedTask, \ from calculate.lib.utils.portage import Layman, EmergeLog, EmergeLogNamedTask, \
InstalledPackageInfo, EbuildInfoError, EbuildInfo, ChrootEix InstalledPackageInfo, EbuildInfoError, EbuildInfo, ChrootEix, \
get_packages_files_directory, get_manifest_files_directory
from calculate.update.emerge_parser import EmergeParser, \ from calculate.update.emerge_parser import EmergeParser, \
EmergeError, ChrootEmergeCommand, ChrootCommandExecutor EmergeError, ChrootEmergeCommand, ChrootCommandExecutor, RevdepPercentBlock, \
EmergeCache
from calculate.lib.cl_log import log
from .build_storage import Build from .build_storage import Build
from calculate.update.update import Update from calculate.update.update import Update
from calculate.install.distr import Distributive from calculate.install.distr import Distributive
@ -49,6 +53,7 @@ class Builder(Update):
self.pretend_package_list = {} self.pretend_package_list = {}
self.update_map = {} self.update_map = {}
self.color_print = get_color_print() self.color_print = get_color_print()
self.emerge_cache = EmergeCache()
def mount_target(self, target): def mount_target(self, target):
dir_distro = target.convertToDirectory() dir_distro = target.convertToDirectory()
@ -185,10 +190,10 @@ class Builder(Update):
clTempl = None clTempl = None
try: try:
clVars.importData() clVars.importData()
# TODO: использование параметра -T
if distro_dv: if distro_dv:
clVars.Set('cl_template_path_use', clVars.Set('cl_template_path_use',
[pathJoin(chroot, x) for x in distro_dv.Get('cl_template_path')], [pathJoin(chroot, x)
for x in distro_dv.Get('cl_template_path')],
force=True) force=True)
clVars.Set('cl_env_path', clVars.Set('cl_env_path',
[pathJoin(chroot, x) for x in clVars.Get('cl_env_path')], [pathJoin(chroot, x) for x in clVars.Get('cl_env_path')],
@ -196,10 +201,10 @@ class Builder(Update):
clVars.Set('cl_make_profile', path.join(chroot, clVars.Set('cl_make_profile', path.join(chroot,
'etc/portage/make.profile'), force=True) 'etc/portage/make.profile'), force=True)
clVars.Set('cl_action', action, force=True) clVars.Set('cl_action', action, force=True)
clVars.Set('cl_verbose_set', self.clVars.Get('cl_verbose_set'))
clVars.Set('cl_chroot_status', 'on', force=True) clVars.Set('cl_chroot_status', 'on', force=True)
#clVars.Set("cl_dispatch_conf", self.clVars.Get('cl_dispatch_conf'), for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
# force=True) "update.cl_update_world"):
clVars.Set(copyvar, self.clVars.Get(copyvar), force=True)
clVars.flIniFile() clVars.flIniFile()
cltFilter=True if cltFilter in (True,"on") else False cltFilter=True if cltFilter in (True,"on") else False
clVars.Set("cl_chroot_path", chroot, True) clVars.Set("cl_chroot_path", chroot, True)
@ -263,8 +268,8 @@ class Builder(Update):
progname, "--repo=%s" % repname, "--update", progname, "--repo=%s" % repname, "--update",
"--jobs=%s" % cpu_num, stderr=STDOUT) "--jobs=%s" % cpu_num, stderr=STDOUT)
def clear_log(self, builder_id): def clear_log(self, builder_id_path):
logname = "build-%s" % builder_id logname = "build-%s" % builder_id_path
mainlog = self.clVars.Get('core.cl_log_path') mainlog = self.clVars.Get('core.cl_log_path')
logpath = path.join(mainlog, logname) logpath = path.join(mainlog, logname)
if path.exists(logpath): if path.exists(logpath):
@ -273,25 +278,25 @@ class Builder(Update):
return True return True
def _get_log_file(self): def _get_log_file(self):
logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id'), logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'),
self.clVars.Get('cl_task_name')) self.clVars.Get('cl_task_name'))
mainlog = self.clVars.Get('core.cl_log_path') mainlog = self.clVars.Get('core.cl_log_path')
return path.join(mainlog, logname) return path.join(mainlog, logname)
def emerge_ask(self, param, *packages): def emerge_ask(self, pretend, *params):
""" """
Вывести информацию об обновлении Вывести информацию об обновлении
""" """
deo = self.clVars.Get('cl_emerge_default_opts') deo = self.clVars.Get('cl_emerge_default_opts')
param = [param] param = [x for x in params if x.startswith("-")]
packages = [x for x in params if not x.startswith("-")]
chroot_path = self.clVars.Get('cl_builder_path') chroot_path = self.clVars.Get('cl_builder_path')
logfile = self._get_log_file() logfile = self._get_log_file()
with EmergeParser(ChrootEmergeCommand(chroot_path, with EmergeParser(ChrootEmergeCommand(chroot_path,
list(packages), list(packages),
emerge_default_opts=deo, emerge_default_opts=deo,
extra_params=param, extra_params=param,
logfile=logfile logfile=logfile)) as emerge:
)) as emerge:
try: try:
emerge.question.action = lambda x: False emerge.question.action = lambda x: False
emerge.run() emerge.run()
@ -299,11 +304,14 @@ class Builder(Update):
emergelike = self.clVars.Get( emergelike = self.clVars.Get(
'update.cl_update_emergelist_set') == 'on' 'update.cl_update_emergelist_set') == 'on'
self._display_install_package(emerge, emergelike) self._display_install_package(emerge, emergelike)
answer = self.askConfirm( if not pretend:
_("Would you like to merge these packages?"), "yes") answer = self.askConfirm(
if answer == "no": _("Would you like to merge these packages?"), "yes")
emerge.command.send("no\n") if answer == "no":
raise KeyboardInterrupt emerge.command.send("no\n")
raise KeyboardInterrupt
else:
return True
else: else:
self.printSUCCESS("Nothing to merge") self.printSUCCESS("Nothing to merge")
except EmergeError as e: except EmergeError as e:
@ -401,11 +409,11 @@ class Builder(Update):
map_rep[pkg['repository']]): map_rep[pkg['repository']]):
yield pkg.atom yield pkg.atom
except EbuildInfoError: except EbuildInfoError:
yield pkg.atom pass
rebuild_list = map(lambda x: "=%s" % x, rebuild_generator()) rebuild_list = map(lambda x: "=%s" % x, rebuild_generator())
if rebuild_list: if rebuild_list:
return self.emerge_ask("-1", *rebuild_list) return self.emerge_ask(False, "-1", *rebuild_list)
return True return True
class Driver(object): class Driver(object):
@ -533,26 +541,35 @@ class Builder(Update):
ef = EmergeFetcher(ChrootEmergeCommand( ef = EmergeFetcher(ChrootEmergeCommand(
builder_path, ["=%s" % x for x in package_list], builder_path, ["=%s" % x for x in package_list],
extra_params=["-Of", "--ask=n"])) extra_params=["-Of", "--ask=n"], logfile="%s.2" % logfile))
try: try:
for package in ef: for package in ef:
pkg_name = str(package) pkg_name = str(package)
if binary_map.get(pkg_name, False): if binary_map.get(pkg_name, False):
for fn in package.files: for fn in package.files:
pkgdir_files.append("%s/%s"%(package['CATEGORY'], pkgdir_files.append("%s/%s" % (package['CATEGORY'],
fn)) fn))
else: else:
for fn in package.files: for fn in package.files:
distdir_files.append(fn) distdir_files.append(fn)
except EmergeFetcherError: if ef.failed():
raise BuilderError(_("Failed to get %s") % drv_name)
except EmergeFetcherError as e:
if e.extension:
self.printPre("\n%s\n"%e.extension)
if e.errno == EmergeFetcherError.FetchErrno.NeedManually:
raise BuilderError(
_("Failed to fetch files for %s") % drv_name)
repeat_driver_list.append([drv_name, drv_mask, drv_atom]) repeat_driver_list.append([drv_name, drv_mask, drv_atom])
if not driver_list and repeat_driver_list: if not driver_list and repeat_driver_list:
driver_list = repeat_driver_list driver_list = repeat_driver_list
repeat_driver_list = [] repeat_driver_list = []
self.printWARNING(_("Waiting for unlock %s") self.printWARNING(_("Waiting for unlock %s")
% driver_list[0][0]) % driver_list[0][0])
time.sleep(10) time.sleep(10)
self.startTask(_("Cleaning and copying driver files")) self.startTask(_("Cleaning and copying driver files"))
for source_dn, source, target_dn, target in [ for source_dn, source, target_dn, target in [
(builder_distdir, distdir_files, (builder_distdir, distdir_files,
@ -648,6 +665,11 @@ class Builder(Update):
return True return True
def check_obsolete(self, builder_path): def check_obsolete(self, builder_path):
"""
Проверка на устаревшие установленные пакеты
:param builder_path:
:return:
"""
chroot_eix = ChrootEix(builder_path, [], ChrootEix.Option.TestObsolete) chroot_eix = ChrootEix(builder_path, [], ChrootEix.Option.TestObsolete)
l = chroot_eix.get_packages() l = chroot_eix.get_packages()
if l: if l:
@ -673,3 +695,104 @@ class Builder(Update):
logfile=logfile)) as emerge: logfile=logfile)) as emerge:
self._startEmerging(emerge) self._startEmerging(emerge)
return True return True
def revdep_rebuild(self, builder_path, cmd, *params):
"""
Запуск revdep-rebulid
"""
cmd_path = self.get_prog_path(cmd)
logfile = self._get_log_file()
if not cmd_path:
raise BuilderError(_("Failed to find the %s command") % cmd)
with EmergeParser(ChrootCommandExecutor(
builder_path, cmd_path, params, logfile=logfile)) as emerge:
revdep = RevdepPercentBlock(emerge)
self.addProgress()
revdep.add_observer(self.setProgress)
revdep.action = lambda x: (
self.endTask(), self.startTask(_("Assigning files to packages"))
if "Assign" in revdep else None)
self._startEmerging(emerge)
return True
def syncLaymanRepository(self, repname):
"""
Обновить репозиторий через layman
"""
chroot_path = self.clVars.Get('cl_builder_path')
layman = self.get_prog_path('/usr/bin/layman')
if not layman:
raise BuilderError(_("The Layman tool is not found"))
rpath = self.clVars.Select('cl_builder_other_rep_path',
where='cl_builder_other_rep_name',
eq=repname, limit=1)
laymanname = path.basename(rpath)
self.stash_cache(rpath, laymanname)
try:
if Git.is_git(rpath):
self.addProgress()
p = PercentProgress(
"/usr/bin/chroot", chroot_path,
layman, "-s", laymanname, part=1, atty=True)
for perc in p.progress():
self.setProgress(perc)
else:
p = process(
"/usr/bin/chroot", chroot_path,
layman, "-s", repname, stderr=STDOUT)
if p.failed():
raise BuilderError(
_("Failed to update the {rname} repository").format(
rname=repname),
addon=p.read())
finally:
self.unstash_cache(rpath, laymanname)
return True
def _update_binhost_packages(self):
"""
Выполнить команду обновления файла binhost (Packages.gz)
:return:
"""
chroot_path = self.clVars.Get('cl_builder_path')
os.system('/usr/bin/chroot %s /usr/sbin/emaint binhost -f &>/dev/null'%
chroot_path)
def cleanpkg(self):
"""
Очистка системы от устаревших distfiles и packages
:return:
"""
builder_path = self.clVars.Get('cl_builder_path')
portdirs = [
pathJoin(builder_path, x)
for x in self.clVars.Get("builder.cl_builder_repository_location")]
pkgfiles = get_packages_files_directory(*portdirs)
distdirfiles = get_manifest_files_directory(*portdirs)
distdir = self.clVars.Get('builder.cl_builder_linux_distdir')
pkgdir = self.clVars.Get('builder.cl_builder_linux_pkgdir')
logfile = self._get_log_file()
logger = log("update_cleanpkg.log", filename=logfile,
formatter="%(asctime)s - %(clean)s - %(message)s")
return self._cleanpkg(
distdir, pkgdir, distdirfiles, pkgfiles, logger)
def raiseOutdate(self):
"""
Установить флаг данные о репозиториях устарели (необходим для выполнения
eix-update и прочих команд обновляющих кэш
:return:
"""
self.clVars.Set('cl_builder_outdate_set', 'on', force=True)
def apply_branch_variables(self):
"""
Применить значение переменной для выбора веток репозиториев
при обновлении
"""
self.clVars.Set('update.cl_update_branch_name',
self.clVars.Get('builder.cl_builder_branch_name'))
return True

39
pym/builder/emerge_fetch.py
Unescape View File

@ -28,7 +28,15 @@ class EmergePackageFetch(EmergePackage):
files = [] files = []
class EmergeFetcherError(Exception): class EmergeFetcherError(Exception):
pass class FetchErrno:
Generic = 0
Lock = 1
NeedManually = 2
def __init__(self, message, errno=FetchErrno.Generic, extension=""):
super(EmergeFetcherError, self).__init__(message)
self.errno = errno
self.extension = extension
class EmergeFetcher(object): class EmergeFetcher(object):
_color_block = EmergeInformationBlock._color_block _color_block = EmergeInformationBlock._color_block
@ -40,6 +48,9 @@ class EmergeFetcher(object):
re_filename = re.compile("^{c} [*] {c}(\S+).*;-\)".format(c=_color_block), re_filename = re.compile("^{c} [*] {c}(\S+).*;-\)".format(c=_color_block),
re.M) re.M)
lock_token = "is already locked by another fetcher"
manually_token = "to be downloaded manually"
def __init__(self, emerge_command): def __init__(self, emerge_command):
""" """
:param package_list: список пакетов типа (EmergeUpdateInfo) :param package_list: список пакетов типа (EmergeUpdateInfo)
@ -48,9 +59,24 @@ class EmergeFetcher(object):
self.emerge_command = emerge_command self.emerge_command = emerge_command
def parse(self, data): def parse(self, data):
if "is already locked by another fetcher" in data: if self.lock_token in data:
raise EmergeFetcherError(
_("File is already locked by another fetcher."),
errno=EmergeFetcherError.FetchErrno.Lock)
if self.manually_token in data:
extension = re.search(
"{nl}( {c}\*{c} The driver.*to be downloaded.*?){nl}{nl}".format(
c=self._color_block,
nl=self._new_line,
), data, re.S)
if extension:
extension = extension.group(1)
else:
extension = ""
raise EmergeFetcherError( raise EmergeFetcherError(
_("File is already locked by another fetcher.")) _("File must be downloaded manually."),
errno=EmergeFetcherError.FetchErrno.NeedManually,
extension=extension)
for package, block in self.re_fetching.findall(data): for package, block in self.re_fetching.findall(data):
ep = EmergePackageFetch(package) ep = EmergePackageFetch(package)
ep.files = [x for x in self.re_filename.findall(block)] ep.files = [x for x in self.re_filename.findall(block)]
@ -60,3 +86,10 @@ class EmergeFetcher(object):
child = self.emerge_command.execute() child = self.emerge_command.execute()
for i in self.parse(child.read()): for i in self.parse(child.read()):
yield i yield i
def success(self):
return self.emerge_command.success()
def failed(self):
return self.emerge_command.failed()

95
pym/builder/utils/cl_builder_image.py
Unescape View File

@ -39,39 +39,76 @@ class ClBuilderImageAction(Action):
# список задач для действия # список задач для действия
tasks = [ tasks = [
# закрепить подключенные данные {'name': 'clear_log',
{'name': 'detach', 'method': 'Builder.clear_log(cl_builder_id_path)',
'message': __("Detach target"),
'method': 'Builder.detach_target(cl_builder_target)',
}, },
{'name': 'umount_system', {'name': 'prepare_action',
'message': __("Umount build system resources"), 'method': 'Builder.set_builder_action("prepare")',
'method': 'Builder.umount_system(cl_builder_target)',
'condition': lambda Get: Get('cl_builder_build')
},
{'name': 'prepare_image',
'method': 'Builder.prepare_image(cl_builder_image)',
'condition': lambda Get: isinstance(Get('cl_builder_image'),
IsoDistributive)
},
{'name': 'squash_action',
'method': 'Builder.set_builder_action("squash")',
}, },
{'name': 'apply_template', {'name': 'apply_template',
'message': __("Configuring squash"), 'message': __("Configuring build"),
# наложить шаблоны в развернутый дистрибутив
'method': 'Builder.applyTemplates(cl_builder_target,False,False,None)', 'method': 'Builder.applyTemplates(cl_builder_target,False,False,None)',
}, },
{'name': 'squash_action', # получить видеодрайверы
'method': 'Builder.set_builder_action("iso")', {'name': 'fetch_video_drivers',
'group': __("Fetching video drivers"),
'tasks': [
{'name': 'fetch_drivers',
'method': 'Builder.fetch_drivers(cl_builder_path,'
'cl_builder_linux_distdir,cl_builder_linux_pkgdir)',
'condition': lambda Get: Get('cl_builder_videodrv_set') == 'on'
},
{'name': 'create_video_data',
'message': __("Creating install video driver data"),
'method': 'Builder.create_video_data(cl_builder_path,'
'cl_builder_repository_data)',
'condition': lambda Get: Get('cl_builder_videodrv_set') == 'on'
},
]
}, },
# распаковка дистрибутива {'name': 'remove_video_drivers',
{'name': 'unpack', 'method': 'Builder.remove_video_drivers(cl_builder_path)',
'message': __("Pack squash"), 'condition': lambda Get: Get('cl_builder_videodrv_set') == 'off'
'method': 'Install.unpack(cl_builder_target, cl_builder_image, "0")',
}, },
{'name': 'umount_system!:mount_system', {'name': 'creating_live_image',
'warning': _("Restore build system resources"), 'group': __("Creating live image"),
'method': 'Builder.mount_target(cl_builder_target)', 'tasks': [
'condition': lambda Get: Get('cl_builder_build') # закрепить подключенные данные
}] {'name': 'detach',
'message': __("Detach target"),
'method': 'Builder.detach_target(cl_builder_target)',
},
{'name': 'umount_system',
'message': __("Umount build system resources"),
'method': 'Builder.umount_system(cl_builder_target)',
'condition': lambda Get: Get('cl_builder_build')
},
{'name': 'prepare_image',
'method': 'Builder.prepare_image(cl_builder_image)',
'condition': lambda Get: isinstance(Get('cl_builder_image'),
IsoDistributive)
},
{'name': 'squash_action',
'method': 'Builder.set_builder_action("squash")',
},
{'name': 'apply_template',
'message': __("Configuring squash"),
# наложить шаблоны в развернутый дистрибутив
'method': 'Builder.applyTemplates(cl_builder_target,'
'False,False,None)',
},
{'name': 'squash_action',
'method': 'Builder.set_builder_action("iso")',
},
# распаковка дистрибутива
{'name': 'unpack',
'message': __("Pack squash"),
'method': 'Install.unpack(cl_builder_target, cl_builder_image, "0")',
},
{'name': 'umount_system!:mount_system',
'warning': _("Restore build system resources"),
'method': 'Builder.mount_target(cl_builder_target)',
'condition': lambda Get: Get('cl_builder_build')
}]
}
]

34
pym/builder/utils/cl_builder_profile.py
Unescape View File

@ -71,25 +71,16 @@ class ClBuilderProfileAction(Action):
'foreach': 'cl_builder_profile_sync_rep', 'foreach': 'cl_builder_profile_sync_rep',
'message': __("Syncing the {eachvar:capitalize} repository"), 'message': __("Syncing the {eachvar:capitalize} repository"),
'method': 'Builder.syncRepositories(eachvar)', 'method': 'Builder.syncRepositories(eachvar)',
#'condition': lambda Get: Get('cl_update_profile_sync_rep')
}, },
{'name': 'regen_cache', {'name': 'regen_cache',
'foreach': 'cl_builder_sync_overlay_rep', 'foreach': 'cl_builder_sync_overlay_rep',
'message': __("Updating the {eachvar:capitalize} repository cache"), 'message': __("Updating the {eachvar:capitalize} repository cache"),
'essential': False, 'essential': False,
'method': 'Builder.regenCache(eachvar)', 'method': 'Builder.regenCache(eachvar)',
#'condition': (
# lambda Get: (Get('cl_update_outdate_set') == 'on' and
# Get('cl_update_metadata_force') != 'skip' or
# Get('cl_update_metadata_force') == 'force'))
}, },
{'name': 'eix_update', {'name': 'eix_update',
'message': __("Updating the eix cache"), 'message': __("Updating the eix cache"),
'method': 'Builder.eixUpdate(cl_builder_repository_name)', 'method': 'Builder.eixUpdate(cl_builder_repository_name)',
#'condition': (
# lambda Get: (Get('cl_update_outdate_set') == 'on' and
# Get('cl_update_eixupdate_force') != 'skip' or
# Get('cl_update_eixupdate_force') == 'force'))
}, },
# save # save
{'name': 'save', {'name': 'save',
@ -115,19 +106,20 @@ class ClBuilderProfileAction(Action):
'method': 'Builder.set_profile(' 'method': 'Builder.set_profile('
'cl_builder_profile_system_shortname,cl_builder_path)' 'cl_builder_profile_system_shortname,cl_builder_path)'
}, },
{'name': 'merge_utils', #{'name': 'merge_utils',
'message': __("Calculating dependencies"), # 'message': __("Calculating dependencies"),
'method': 'Builder.emerge("-console -pxe","-1","sys-apps/calculate-utils")', # 'method': 'Builder.emerge("-console -pxe","-1","sys-apps/calculate-utils")',
# TODO: возможно добавить параметр необходимости установки # # TODO: возможно добавить параметр необходимости установки
'condition': lambda Get: not isPkgInstalled( # 'condition': lambda Get: not isPkgInstalled(
'sys-apps/calculate-utils', Get('cl_builder_path')) # 'sys-apps/calculate-utils', Get('cl_builder_path'))
}, # },
{'name': 'reconfigure', {'name': 'reconfigure',
'message': __("Reconfigure the settings"), 'message': __("Reconfigure the settings"),
'method': 'Builder.apply_templates(cl_builder_path,' 'method': 'Builder.apply_templates(cl_builder_path,'
'cl_template_clt_set,True,None,False,"merge",' 'cl_template_clt_set,True,None,False,"merge",'
'cl_builder_profile_datavars)', 'cl_builder_profile_datavars)',
'condition': lambda Get: Get('cl_templates_locate') 'condition': lambda Get: (
Get('update.cl_update_skip_setup_set') == 'off')
}, },
{'name': 'fix_settings', {'name': 'fix_settings',
'message': __("Fixing the settings"), 'message': __("Fixing the settings"),

107
pym/builder/utils/cl_builder_update.py
Unescape View File

@ -39,8 +39,8 @@ class ClBuilderUpdateAction(Action):
native_error = (DistributiveError, FilesError, UpdateError, native_error = (DistributiveError, FilesError, UpdateError,
BuilderError, GitError, EmergeError) BuilderError, GitError, EmergeError)
successMessage = __("The system was successfully updated") successMessage = None
failedMessage = __("Failed to update the system") failedMessage = None
interruptMessage = __("System update manually interrupted") interruptMessage = __("System update manually interrupted")
def was_installed(pkg, task_name): def was_installed(pkg, task_name):
@ -62,7 +62,7 @@ class ClBuilderUpdateAction(Action):
# список задач для действия # список задач для действия
tasks = [ tasks = [
{'name': 'clear_log', {'name': 'clear_log',
'method': 'Builder.clear_log(cl_builder_id)', 'method': 'Builder.clear_log(cl_builder_id_path)',
}, },
{'name': 'apply_template', {'name': 'apply_template',
'message': __("Configuring build"), 'message': __("Configuring build"),
@ -75,24 +75,49 @@ class ClBuilderUpdateAction(Action):
'method': 'Builder.reconfigureProfileVars(cl_builder_linux_datavars,' 'method': 'Builder.reconfigureProfileVars(cl_builder_linux_datavars,'
'cl_builder_path)' 'cl_builder_path)'
}, },
{'name': 'apply_branch_vars',
'method': 'Builder.apply_branch_variables()'
},
{'name': 'reps_synchronization', {'name': 'reps_synchronization',
'group': __("Repositories synchronization"), 'group': __("Repositories synchronization"),
'tasks': [ 'tasks': [
{'name': 'sync_reps', {'name': 'sync_reps',
'foreach': 'cl_builder_profile_sync_rep', 'foreach': 'cl_builder_sync_rep',
'message': __("Syncing the {eachvar:capitalize} repository"), 'message': __("Syncing the {eachvar:capitalize} repository"),
'method': 'Builder.syncRepositories(eachvar)', 'method': 'Builder.syncRepositories(eachvar)',
}, },
{'name': 'sync_other_reps',
'foreach': 'builder.cl_builder_other_rep_name',
'message': __("Syncing the {eachvar:capitalize} repository"),
'method': 'Builder.syncLaymanRepository(eachvar)',
'condition': lambda Get: Get('update.cl_update_other_set') == 'on'
},
{'name': 'regen_cache', {'name': 'regen_cache',
'foreach': 'cl_builder_sync_overlay_rep', 'foreach': 'cl_builder_sync_overlay_rep',
'message': __("Updating the {eachvar:capitalize} repository cache"), 'message': __("Updating the {eachvar:capitalize} repository cache"),
'essential': False, 'essential': False,
'method': 'Builder.regenCache(eachvar)', 'method': 'Builder.regenCache(eachvar)',
'condition': (lambda Get: (
Get('builder.cl_builder_outdate_set') == 'on' and
Get('update.cl_update_egencache_force') != 'skip' or
Get('update.cl_update_egencache_force') == 'force'))
}, },
{'name': 'eix_update', {'name': 'eix_update',
'message': __("Updating the eix cache"), 'message': __("Updating the eix cache"),
'method': 'Builder.eixUpdate(cl_builder_repository_name)', 'method': 'Builder.eixUpdate(cl_builder_repository_name)',
'condition': (lambda Get: (
Get('builder.cl_builder_outdate_set') == 'on' and
Get('update.cl_update_eixupdate_force') != 'skip' or
Get('update.cl_update_eixupdate_force') == 'force'))
}, },
{'name': 'sync_reps:cleanpkg',
'message': __("Removing obsolete distfiles and binary packages"),
'method': 'Builder.cleanpkg()',
'condition': lambda Get: (
Get('builder.cl_builder_outdate_set') == 'on' and
Get('update.cl_update_cleanpkg_set') == 'on'),
'essential': False
},
# сообщение удачного завершения при обновлении репозиториев # сообщение удачного завершения при обновлении репозиториев
{'name': 'success_syncrep', {'name': 'success_syncrep',
'message': __("Synchronization finished"), 'message': __("Synchronization finished"),
@ -125,12 +150,20 @@ class ClBuilderUpdateAction(Action):
'tasks': [ 'tasks': [
{'name': 'update_world', {'name': 'update_world',
'message': __("Calculating dependencies"), 'message': __("Calculating dependencies"),
'method': 'Builder.emerge_ask("-uDN", "--changed-deps",' 'method': 'Builder.emerge_ask(update.cl_update_pretend_set,'
'"--with-bdeps=y", "@world")' '"-uDN","--changed-deps",'
'"--with-bdeps=y","@world")'
} }
] ],
'condition': lambda Get: (
Get('update.cl_update_sync_only_set') == 'off')
},
{'name': 'update_other',
'condition': lambda Get: (
Get('update.cl_update_pretend_set') == 'off' and
Get('update.cl_update_sync_only_set') == 'off')
}, },
{'name': 'group_changed_packages', {'name': 'update_other:group_changed_packages',
'group': __("Rebuild modified packages"), 'group': __("Rebuild modified packages"),
'tasks': [ 'tasks': [
{'name': 'changed_packages', {'name': 'changed_packages',
@ -140,7 +173,7 @@ class ClBuilderUpdateAction(Action):
} }
] ]
}, },
{'name': 'update_python', {'name': 'update_other:update_python',
'group': __("Updating Python"), 'group': __("Updating Python"),
'tasks': [ 'tasks': [
{'name': 'python_updater', {'name': 'python_updater',
@ -154,7 +187,7 @@ class ClBuilderUpdateAction(Action):
}, },
] ]
}, },
{'name': 'update_perl', {'name': 'update_other:update_perl',
'group': __("Updating Perl"), 'group': __("Updating Perl"),
'tasks': [ 'tasks': [
{'name': 'perl_cleaner', {'name': 'perl_cleaner',
@ -168,7 +201,7 @@ class ClBuilderUpdateAction(Action):
}, },
] ]
}, },
{'name': 'depclean', {'name': 'update_other:depclean',
'group': __("Cleaning the system from needless packages"), 'group': __("Cleaning the system from needless packages"),
'tasks': [ 'tasks': [
{'name': 'update_depclean', {'name': 'update_depclean',
@ -204,44 +237,46 @@ class ClBuilderUpdateAction(Action):
'decoration': 'Builder.update_task("%s")' % 'decoration': 'Builder.update_task("%s")' %
EmergeMark.PreservedLibs EmergeMark.PreservedLibs
}, },
{'name': 'update_world:revdev_rebuild',
'message': __('Checking reverse dependencies'),
'method': 'Builder.revdep_rebuild(cl_builder_path,'
'"revdep-rebuild")',
'condition': lambda Get: (
Get('update.cl_update_skip_rb_set') == 'off' and
ClBuilderUpdateAction.was_installed(
'.*', EmergeMark.RevdepRebuild)(Get)),
'decoration': 'Builder.update_task("%s")' %
EmergeMark.RevdepRebuild
},
{'name': 'update_world:dispatch_conf_end', {'name': 'update_world:dispatch_conf_end',
'message': __("Updating configuration files"), 'message': __("Updating configuration files"),
'method':'Builder.dispatchConf()', 'method': 'Builder.dispatchConf()',
'condition': lambda Get: Get('cl_dispatch_conf') != 'skip' 'condition': lambda Get: Get('cl_dispatch_conf') != 'skip'
}, },
] ],
'depend': Tasks.has("update_other")
}, },
{'name': 'prelink', {'name': 'update_other:prelink',
'message': __("Executing prelink"), 'message': __("Executing prelink"),
'method': 'Builder.chroot_command(cl_builder_path,"prelink",' 'method': 'Builder.chroot_command(cl_builder_path,"prelink",'
'"-afmR")', '"-afmR")',
'condition': was_installed('.*', EmergeMark.Prelink), 'condition': was_installed('.*', EmergeMark.Prelink),
'decoration': 'Builder.update_task("%s")' % EmergeMark.Prelink 'decoration': 'Builder.update_task("%s")' % EmergeMark.Prelink
}, },
{'name': 'fetch_video_drivers', {'name': 'update_other:reading_news',
'group': __("Fetching video drivers"),
'tasks': [
{'name': 'fetch_drivers',
'method': 'Builder.fetch_drivers(cl_builder_path,'
'cl_builder_linux_distdir,cl_builder_linux_pkgdir)',
'condition': lambda Get: Get('cl_builder_videodrv_set') == 'on'
},
{'name': 'create_video_data',
'message': __("Creating install video driver data"),
'method': 'Builder.create_video_data(cl_builder_path,'
'cl_builder_repository_data)',
'condition': lambda Get: Get('cl_builder_videodrv_set') == 'on'
},
]
},
{'name': 'remove_video_drivers',
'method': 'Builder.remove_video_drivers(cl_builder_path)',
'condition': lambda Get: Get('cl_builder_videodrv_set') == 'off'
},
{'name': 'reading_news',
'method': 'Builder.reading_news(cl_builder_path)' 'method': 'Builder.reading_news(cl_builder_path)'
}, },
{'name': 'check_obsolete', {'name': 'update_other:check_obsolete',
'method': 'Builder.check_obsolete(cl_builder_path)' 'method': 'Builder.check_obsolete(cl_builder_path)'
} }
] + [
{'name': 'failed',
'error': __("Failed to update the system"),
'depend': (Tasks.failed() & Tasks.hasnot("interrupt"))
},
# сообщение удачного завершения при обновлении ревизии
{'name': 'update_other:success_rev',
'message': __("The system was successfully updated"),
}
] ]
was_installed = staticmethod(was_installed)

9
pym/builder/variables/action.py
Unescape View File

@ -35,6 +35,7 @@ class Actions:
ImageSquash = "squash" ImageSquash = "squash"
ImageIso = "iso" ImageIso = "iso"
PrepareAssemble = "prepare"
class VariableAcBuilderSquash(ReadonlyVariable): class VariableAcBuilderSquash(ReadonlyVariable):
""" """
@ -69,6 +70,10 @@ class VariableAcBuilderSetup(ReadonlyVariable):
if (action in (Actions.Prepare, Actions.Update) and if (action in (Actions.Prepare, Actions.Update) and
not self.GetBool('cl_builder_stage_set')): not self.GetBool('cl_builder_stage_set')):
return "on" return "on"
image_action = self.Get('cl_builder_action')
if (action in (Actions.Image,) and
image_action == Actions.PrepareAssemble):
return "on"
return "off" return "off"
@ -80,4 +85,8 @@ class VariableAcBuilderPrepare(ReadonlyVariable):
action = self.Get('cl_action') action = self.Get('cl_action')
if action in (Actions.Prepare, Actions.Update): if action in (Actions.Prepare, Actions.Update):
return "on" return "on"
image_action = self.Get('cl_builder_action')
if (action in (Actions.Image,) and
image_action == Actions.PrepareAssemble):
return "on"
return "off" return "off"

128
pym/builder/variables/builder.py
Unescape View File

@ -18,6 +18,9 @@ import sys
from os import path from os import path
import os import os
import re import re
from calculate.lib.configparser import ConfigParser
from calculate.lib.utils.git import Git
from calculate.lib.utils.git import GitError
from .action import Actions from .action import Actions
from calculate.install import distr from calculate.install import distr
from calculate.lib.utils.device import getUdevDeviceInfo, humanreadableSize from calculate.lib.utils.device import getUdevDeviceInfo, humanreadableSize
@ -27,7 +30,8 @@ from calculate.lib.utils.kernel import InitrdFile
from calculate.lib.utils.tools import max_default from calculate.lib.utils.tools import max_default
from ..build_storage import BuildStorage, Build from ..build_storage import BuildStorage, Build
from ..drive_spool import DriveSpool from ..drive_spool import DriveSpool
from calculate.lib.datavars import Variable, VariableError, ReadonlyVariable from calculate.lib.datavars import Variable, VariableError, ReadonlyVariable, \
TableVariable
from functools import wraps from functools import wraps
_ = lambda x:x _ = lambda x:x
@ -303,7 +307,9 @@ class VariableClBuilderTarget(Variable):
if not builder_disk_dev: if not builder_disk_dev:
return "" return ""
build_id = self.Get('cl_builder_id') build_id = self.Get('cl_builder_id')
mount_dir = path.join(distr.DefaultMountPath.BaseMountPath, build_id) build_id_path = self.Get('cl_builder_id_path')
mount_dir = path.join(distr.DefaultMountPath.BaseMountPath,
build_id_path)
source = self.Get('cl_builder_source_filename') source = self.Get('cl_builder_source_filename')
if self.GetBool('cl_builder_layered_set'): if self.GetBool('cl_builder_layered_set'):
dist_obj = distr.LayeredDistributive(mount_dir, builder_disk_dev, dist_obj = distr.LayeredDistributive(mount_dir, builder_disk_dev,
@ -394,6 +400,9 @@ class VariableClBuilderNewId(BaseBuildId):
value = "" value = ""
untrusted = True untrusted = True
def get(self):
return self.Get('cl_builder_profile_name')
def check(self, value): def check(self, value):
if not value: if not value:
raise VariableError(_("Please specify the id")) raise VariableError(_("Please specify the id"))
@ -485,6 +494,15 @@ class VariableClBuilderBrokenId(BaseBuildId):
if not l: if not l:
raise VariableError(_("Assemble %s is not found") % value) raise VariableError(_("Assemble %s is not found") % value)
class VariableClBuilderIdPath(ReadonlyVariable):
"""
Преобразование сборки id в имя походящее для путей
"""
def get(self):
build_id = self.Get('cl_builder_id')
if build_id:
return re.sub("[/:]", "_", self.Get('cl_builder_id'))
return ""
class VariableClBuilderId(ReadonlyVariable): class VariableClBuilderId(ReadonlyVariable):
""" """
@ -550,7 +568,7 @@ class VariableClBuilderPkgdir(Variable):
""" """
def fallback(self): def fallback(self):
return path.join(self.Get('cl_builder_base_path'), return path.join(self.Get('cl_builder_base_path'),
self.Get('cl_builder_id'), "packages") self.Get('cl_builder_id_path'), "packages")
def get(self): def get(self):
action = self.Get('cl_action') action = self.Get('cl_action')
@ -570,7 +588,7 @@ class VariableClBuilderAction(ReadonlyVariable):
value = "" value = ""
class VariableClBuilderImageFilename(ReadonlyVariable): class VariableClBuilderImageFilename(Variable):
""" """
Название iso образа Название iso образа
""" """
@ -585,13 +603,14 @@ class VariableClBuilderImageFilename(ReadonlyVariable):
def get(self): def get(self):
base_dn = self.Get('cl_builder_base_path') base_dn = self.Get('cl_builder_base_path')
build_id = self.Get('cl_builder_id') build_id = self.Get('cl_builder_id')
build_id_path = self.Get('cl_builder_id_path')
if build_id: if build_id:
shortname = self.Get('os_builder_linux_shortname') shortname = self.Get('os_builder_linux_shortname').lower()
buildnumber = self.Get('os_builder_linux_build') buildnumber = self.Get('os_builder_linux_build')
arch = self.Get('os_builder_arch_machine') arch = self.Get('os_builder_arch_machine')
imagename = "%s-%s-%s.iso" % (shortname, buildnumber, imagename = "%s-%s-%s.iso" % (shortname, buildnumber,
arch) arch)
return path.join(base_dn, build_id, "linux", imagename) return path.join(base_dn, build_id_path, "linux", imagename)
return "" return ""
@ -609,8 +628,9 @@ class VariableClBuilderIsoPath(ReadonlyVariable):
def get(self): def get(self):
base_dn = self.Get('cl_builder_iso_base_path') base_dn = self.Get('cl_builder_iso_base_path')
build_id = self.Get('cl_builder_id') build_id = self.Get('cl_builder_id')
build_id_path = self.Get('cl_builder_id_path')
if build_id: if build_id:
dn = "iso-%s" % self.Get('cl_builder_id') dn = "iso-%s" % build_id_path
directory = path.join(base_dn, dn) directory = path.join(base_dn, dn)
new_dn = directory new_dn = directory
for i in range(0, 9999): for i in range(0, 9999):
@ -845,23 +865,93 @@ class VariableClBuilderTemplateLocation(Variable):
return [x for x in self.Get('main.cl_template_location') return [x for x in self.Get('main.cl_template_location')
if x not in ('remote', 'local')] if x not in ('remote', 'local')]
class VariableClBuilderBranchName(Variable):
class VariableClBuilderOutdateSet(ReadonlyVariable):
"""
Флаг устанавливаемый в ходе обновления репозиториев,
сообщающий что хотя бы один из запланированных репозиториев
обновлен и следует обновляет различные метаданные
Если обновляются прочие оверлеи - данные считаются что устарели
"""
type = "bool"
def get(self):
if (self.Get('update.cl_update_other_set') == 'on' and
self.Get('cl_builder_other_rep_name')):
return "on"
return "off"
class VariableClBuilderBranch(TableVariable):
"""
Выбор веток репозиториев до которых необходимо обновиться
"""
opt = ["--branch"]
metavalue = 'BRANCHES'
untrusted = True
source = ["cl_builder_branch_rep",
"cl_builder_branch_name"]
def init(self):
self.help = _("set branches for repository (REPOSITORY:BRANCH)")
self.label = _("Repositories branches")
def raiseReadonlyIndexError(self, fieldname="", variablename="", value=""):
"""
Неизвестный оврелей
"""
raise VariableError(_("Repository %s not found") % value)
class VariableClBuilderBranchRep(ReadonlyVariable):
""" """
Ветки оверлеев используемые для сборки системы Список доступных репозиториев
""" """
type = "list" type = "list"
def init(self):
self.label = _("Repositories")
def get(self): def get(self):
action = self.Get('cl_action')
default_branch = Build.Branches.Stable
if action == Actions.ChangeProfile:
pass
else:
build = self.Get('cl_builder_build')
if build and build.references:
return build.references
dv = self.Get('cl_builder_linux_datavars') dv = self.Get('cl_builder_linux_datavars')
if dv: if dv:
return map(lambda x: default_branch, return dv.Get('cl_update_rep_name')
dv.Get('cl_update_rep_name'))
return [] return []
class VariableClBuilderBranchName(Variable):
"""
Список доступных репозиторием
"""
type = "choiceedit-list"
def init(self):
self.label = _("Branches")
def choice(self):
return ["master", "develop", "update"]
def get(self):
dv = self.Get('cl_builder_linux_datavars')
cp = ConfigParser()
cp.read_string(unicode(
dv.Get('update.cl_update_binhost_revisions')[0]))
repo_tag = dv.GetBool('update.cl_update_repo_tag_set')
def generateBranch():
git = Git()
for reppath, repname in zip(dv.Get('cl_update_rep_path'),
dv.Get('cl_update_rep_name')):
tag = cp.get("vcs", repname, fallback="master")
try:
if (repo_tag or git.isTagRepository(
reppath) or not git.is_git(reppath)):
yield tag
else:
yield git.getBranch(reppath) or tag
except GitError:
yield tag
return list(generateBranch())

47
pym/builder/variables/linux.py
Unescape View File

@ -23,6 +23,7 @@ from calculate.install.distr import DistributiveError
from calculate.lib.datavars import ReadonlyVariable, SimpleDataVars, \ from calculate.lib.datavars import ReadonlyVariable, SimpleDataVars, \
VariableError, ReadonlyTableVariable, FieldValue VariableError, ReadonlyTableVariable, FieldValue
from calculate.lib.utils.files import pathJoin from calculate.lib.utils.files import pathJoin
from calculate.lib.utils.portage import Layman
from calculate.lib.utils.tools import ignore from calculate.lib.utils.tools import ignore
from calculate.lib.variables import linux from calculate.lib.variables import linux
from calculate.lib.variables import env from calculate.lib.variables import env
@ -76,6 +77,16 @@ class DataVarsBuilderLinux(linux.LinuxDataVars):
update.VariableClUpdateLaymanConfig(section="update"), update.VariableClUpdateLaymanConfig(section="update"),
update.VariableClUpdateLaymanStorage(section="update"), update.VariableClUpdateLaymanStorage(section="update"),
update.VariableClProfileRepository(section="update"), update.VariableClProfileRepository(section="update"),
update.VariableClUpdateBinhost(section="update"),
update.VariableClUpdateBinhostData(section="update"),
update.VariableClUpdateBinhostHost(section="update"),
update.VariableClUpdateBinhostRecheckSet(section="update"),
update.VariableClUpdateBinhostRevisions(section="update"),
update.VariableClUpdateBinhostTime(section="update"),
update.VariableClUpdateBinhostTimeout(section="update"),
update.VariableClUpdateBinhosts(section="update"),
update.VariableClUpdateRepoTagSet(section="update"),
update.VariableClUpdateRevisionPath(section="update"),
system.VariableOsArchMachineGentoo()] system.VariableOsArchMachineGentoo()]
@classmethod @classmethod
@ -208,6 +219,42 @@ class VariableOsBuilderLinuxSubname(BuilderLinux):
variable = "os_linux_subname" variable = "os_linux_subname"
class VariableClBuilderPortdirOverlay(ReadonlyVariable):
"""
Пути оверлеев в собираемом образе относительно точки монтирования
"""
def get(self):
return [location
for name, location in
filter(None,self.Get('builder.cl_builder_repository_data'))
if name not in ("portage", "gentoo")]
class VariableClBuilderOtherRepData(update.VariableClUpdateOtherRepData):
"""
Информация о прочих репозиториях
"""
source = ['builder.cl_builder_other_rep_name',
'builder.cl_builder_other_rep_path']
portdir_overlay = "builder.cl_builder_portdir_overlay"
class VariableClBuilderOtherRepName(FieldValue,ReadonlyVariable):
"""
Список имен прочих репозиториев
"""
type = "list"
source_variable = "builder.cl_builder_other_rep_data"
column = 0
class VariableClBuilderOtherRepPath(FieldValue,ReadonlyVariable):
"""
Список путей до прочих репозиториев
"""
type = "list"
source_variable = "builder.cl_builder_other_rep_data"
column = 1
class VariableClBuilderRepositoryData(ReadonlyTableVariable): class VariableClBuilderRepositoryData(ReadonlyTableVariable):
""" """
""" """

12
pym/builder/variables/profiles.py
Unescape View File

@ -147,7 +147,6 @@ class VariableClBuilderTemplatesLocate(update.VariableClUpdateTemplatesLocate):
profile_datevars = "builder.cl_builder_profile_datavars" profile_datevars = "builder.cl_builder_profile_datavars"
class VariableClBuilderProfileDatavars(update.VariableClUpdateProfileDatavars): class VariableClBuilderProfileDatavars(update.VariableClUpdateProfileDatavars):
type = "object" type = "object"
@ -233,6 +232,17 @@ class VariableClBuilderProfileSyncRep(ReadonlyVariable):
def get(self): def get(self):
return list(reversed(self.Get('update.cl_update_rep_name'))) return list(reversed(self.Get('update.cl_update_rep_name')))
class VariableClBuilderSyncRep(update.VariableClUpdateSyncRep):
"""
Обновляемый репозиторий
"""
@property
def rep_name(self):
dv = self.Get('cl_builder_linux_datavars')
if dv:
return dv.Get('cl_update_rep_name')
return ""
class VariableClBuilderRemoveRep(ReadonlyVariable): class VariableClBuilderRemoveRep(ReadonlyVariable):
def get(self): def get(self):
dv_builder_linux = self.Get('cl_builder_linux_datavars') dv_builder_linux = self.Get('cl_builder_linux_datavars')

19
pym/builder/wsdl_builder.py
Unescape View File

@ -155,9 +155,19 @@ class Wsdl(WsdlBase):
# описание груп (список лямбда функций) # описание груп (список лямбда функций)
'groups': [ 'groups': [
lambda group: group(_("Update the assemble"), lambda group: group(_("Update the assemble"),
normal=('cl_builder_prepared_id', normal=('cl_builder_prepared_id',),
'cl_builder_videodrv_set',), expert=('update.cl_update_pretend_set',
expert=('cl_templates_locate', 'update.cl_update_sync_only_set',
'update.cl_update_emergelist_set',
'update.cl_update_world',
'update.cl_update_skip_rb_set',
'update.cl_update_other_set',
'update.cl_update_egencache_force',
'update.cl_update_eixupdate_force',
'update.cl_update_cleanpkg_set',
'cl_builder_branch',
'cl_builder_sync_rep',
'cl_templates_locate',
'cl_verbose_set', 'cl_dispatch_conf'), 'cl_verbose_set', 'cl_dispatch_conf'),
next_label=_("Perform"))] next_label=_("Perform"))]
}, },
@ -226,7 +236,8 @@ class Wsdl(WsdlBase):
'groups': [ 'groups': [
lambda group: group(_("Create the image"), lambda group: group(_("Create the image"),
normal=('cl_builder_prepared_id', normal=('cl_builder_prepared_id',
'cl_builder_image_filename'), 'cl_builder_image_filename',
'cl_builder_videodrv_set'),
brief=('cl_builder_kernel_ver', brief=('cl_builder_kernel_ver',
'cl_builder_initrd_install', 'cl_builder_initrd_install',
'os_builder_linux_shortname', 'os_builder_linux_shortname',

Write Preview
Loading…
Cancel
Save