From fedfdacca29b6c5e02f6cd15bd46280b2ea3a5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B8=D1=80=D0=B5=D1=86=D0=BA=D0=B8=D0=B9=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= Date: Thu, 23 Aug 2018 11:54:47 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B0?= =?UTF-8?q?=20=D0=BF=D0=BE=D0=B4=D0=BF=D0=B8=D1=81=D0=B8=20Packages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/builder/builder.py | 86 +++++++++++++--- pym/builder/utils/cl_builder_update.py | 136 +++---------------------- pym/builder/variables/builder.py | 27 +++-- pym/builder/variables/linux.py | 70 ++++++++++++- pym/builder/wsdl_builder.py | 9 +- 5 files changed, 168 insertions(+), 160 deletions(-) diff --git a/pym/builder/builder.py b/pym/builder/builder.py index e3174e7..c2f0373 100644 --- a/pym/builder/builder.py +++ b/pym/builder/builder.py @@ -25,7 +25,7 @@ import json from calculate.core.server.gen_pid import search_worked_process2 from calculate.core.setup_package import ChainProgressTemplate from calculate.lib.cl_template import templateFunction, SystemIni -from calculate.lib.datavars import DataVars, Variable +from calculate.lib.datavars import DataVars, Variable, DataVarsError from calculate.lib.utils.colortext import (TextState, get_color_print) from calculate.lib.utils.common import CmdlineParams, getTupleVersion from calculate.lib.utils.portage import (BinaryPackage, get_binary_file, @@ -60,7 +60,7 @@ from calculate.update.emerge_parser import (EmergeParser, from calculate.lib.cl_log import log from calculate.update.update_tasks import EmergeMark from .build_storage import Build -from calculate.update.update import Update +from calculate.update.update import Update, variable_module from calculate.install.distr import (Distributive, IsoDistributive, DistributiveError, ContainerDistributive) import shutil @@ -102,6 +102,7 @@ class Builder(Update): self.pkgnum = None self.pkgnummax = None self.world_data = "" + self.binhosts_data = None def mount_target(self, target): dir_distro = target.convertToDirectory() @@ -593,6 +594,10 @@ class Builder(Update): with self.clVars.useDefaultModule("update"): return super(Builder, self).regenCache(repname) + def prepare_gpg(self): + with self.clVars.useDefaultModule("update"): + return super(Builder, self).prepare_gpg() + def getGit(self): chroot_path = self.clVars.Get('builder.cl_builder_path') sshkey = pathJoin(chroot_path, @@ -624,6 +629,7 @@ class Builder(Update): makeDirectory(logpath) return True + @variable_module("builder") def _get_log_file(self): logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'), self.clVars.Get('cl_task_name')) @@ -1107,12 +1113,13 @@ class Builder(Update): self._startEmerging(emerge) return True - def update_layman(self, builder_path): + def update_layman(self): """ Обновить базу layman :param builder_path: :return: """ + builder_path = self.clVars.Get("cl_builder_path") cmd = "/usr/bin/layman" cmd_path = self.get_prog_path(cmd) logfile = self._get_log_file() @@ -1310,23 +1317,57 @@ class Builder(Update): raise BuilderError(str(e)) return True - def raiseOutdate(self): + def prepare_update_vars(self): """ - Установить флаг данные о репозиториях устарели (необходим для выполнения - eix-update и прочих команд обновляющих кэш - :return: + Заместить значения update переменных одноимёнными из builder """ - self.clVars.Set('cl_builder_outdate_set', 'on', force=True) + vars_map = { + #Применить значение переменной для выбора веток репозиториев + #при обновлении + 'update.cl_update_branch_name': 'builder.cl_builder_branch_name', - def apply_branch_variables(self): - """ - Применить значение переменной для выбора веток репозиториев - при обновлении - """ - self.clVars.Set('update.cl_update_branch_name', - self.clVars.Get('builder.cl_builder_branch_name')) + 'update.cl_update_gpg_force': 'builder.cl_builder_gpg_force', + 'update.cl_update_gpg_keys': 'builder.cl_builder_gpg_keys', + 'update.cl_update_binhost_revision_path': + 'builder.cl_builder_binhost_revision_path', + 'update.cl_update_binhost_timestamp_path': + 'builder.cl_builder_binhost_timestamp_path', + 'update.cl_update_last_timestamp': + 'builder.cl_builder_last_timestamp', + + 'update.cl_update_binhost_stable_set': + 'builder.cl_builder_binhost_stable_set', + 'update.cl_update_binhost_stable_opt_set': + 'builder.cl_builder_binhost_stable_opt_set', + + 'update.cl_update_binhost': + 'builder.cl_builder_binhost', + + 'update.cl_update_sync_rep': 'builder.cl_builder_sync_rep', + 'update.cl_update_other_rep_name': 'builder.cl_builder_other_rep_name', + 'update.cl_update_usetag_set': 'builder.cl_builder_usetag_set', + + 'update.cl_update_sync_overlay_rep': 'builder.cl_builder_sync_overlay_rep', + 'update.cl_repository_name': 'builder.cl_builder_repository_name' + } + try: + for k,v in vars_map.items(): + self.clVars.Set(k, self.clVars.Get(v), force=True) + except DataVarsError as e: + error = BuilderError(_("Failed to prepare variables for synchronization")) + error.addon = e + raise error return True + @variable_module("builder") + def _get_binhost_logger(self): + logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'), + "binhost-scan.log") + mainlog = self.clVars.Get('core.cl_log_path') + return log("binhost-scan.log", + filename=path.join(mainlog, logname), + formatter="%(message)s") + def isohybrid(self, image_file): """ Преобразовать ISO образ в гибридный @@ -1390,6 +1431,9 @@ class Builder(Update): return False return True + def get_arch_machine(self): + return self.clVars.Get("builder.os_builder_arch_machine") + @property def chroot_process(self): chroot_cmd = getProgPath("/usr/bin/chroot") @@ -2033,6 +2077,8 @@ class Builder(Update): """ self.clVars.Write('cl_update_binhost', binhost) self.clVars.Set('cl_update_package_cache_set', 'on') + self.clVars.Set('cl_update_binhost_list', [binhost], force=True) + self.clVars.Set('cl_update_binhost_unstable_list', [binhost], force=True) return True def index_pkgdir(self, pkgdir, trunkdir, stabledir): @@ -2083,3 +2129,13 @@ class Builder(Update): raise BuilderError( _("Failed to merge prepared binaries: %s") % str(e)) return True + + @variable_module("builder") + def update_binhost_list(self): + self.invalidateVariables('cl_builder_linux_datavars') + self.prepare_update_vars() + return super(Builder,self).update_binhost_list( + self.clVars.Get('cl_builder_linux_datavars')) + + def is_update_action(self, action): + return action == Actions.Update diff --git a/pym/builder/utils/cl_builder_update.py b/pym/builder/utils/cl_builder_update.py index b2fd2bc..7caf906 100644 --- a/pym/builder/utils/cl_builder_update.py +++ b/pym/builder/utils/cl_builder_update.py @@ -29,6 +29,7 @@ from calculate.lib.utils.portage import (EmergeLogNamedTask, PackageList, EmergeLog, isPkgInstalled) from calculate.install.distr import DistributiveError from calculate.update.update_tasks import EmergeMark +from calculate.update.utils.cl_update import get_synchronization_tasks _ = lambda x: x setLocalTranslate('cl_builder3', sys.modules[__name__]) @@ -112,128 +113,19 @@ class ClBuilderUpdateAction(Action): 'method': 'Builder.reconfigureProfileVars(' 'cl_builder_linux_datavars,cl_builder_path)' }, - {'name': 'apply_branch_vars', - 'method': 'Builder.apply_branch_variables()' - }, - {'name': 'reps_synchronization', - 'group': __("Repositories synchronization"), - 'tasks': [ - # запасная синхронизация, в ходе которой ветки - # обновляются до master - {'name': 'sync_reps_fallback', - 'foreach': 'cl_builder_sync_rep', - 'message': - __("Fallback syncing the {eachvar:capitalize} " - "repository"), - 'method': 'Builder.syncRepositories(eachvar,True)', - 'condition': lambda Get: ( - Get('cl_builder_usetag_set') == "on" and - not Get('update.cl_update_binhost_data')[0]) - }, - # обновление переменных информации из binhost - {'name': 'invalidate_vars', - 'method': - 'Builder.invalidateVariables(' - '"cl_builder_linux_datavars")', - 'condition': lambda Get: ( - Get('cl_builder_usetag_set') == "on" and - not Get('update.cl_update_binhost_data')[0]) - }, - {'name': 'binhost_unavailable', - 'error': __("Update server is unavailable"), - 'condition': lambda Get: ( - Get('cl_builder_usetag_set') == "on" and - not Get('update.cl_update_binhost_data')[0]) - }, - {'name': 'sync_reps', - 'foreach': 'cl_builder_sync_rep', - 'message': __("Checking {eachvar:capitalize} updates"), - 'method': 'Builder.syncRepositories(eachvar)', - 'condition': lambda Get: Get('cl_builder_sync_rep') - }, - {'name': 'check_binhost', - 'method': 'Builder.check_binhost(True)', - 'condition': lambda Get: Get( - 'cl_builder_usetag_set') == "on" - }, - {'name': 'use_base_binhost', - 'method': 'Builder.set_base_binhost(' - 'builder.cl_builder_binhost_base)', - 'condition': lambda Get, GetBool: ( - GetBool("builder.cl_builder_binhost_base_set") and - Get("builder.cl_builder_binhost_base")) - }, - {'name': 'update_layman', - 'message': __("Layman cache update"), - 'method': "Builder.update_layman(cl_builder_path)", - 'condition': lambda Get: isPkgInstalled( - "app-portage/layman", prefix=Get('cl_chroot_path')), - 'essential': False, - }, - {'name': 'sync_other_reps', - 'foreach': 'builder.cl_builder_other_rep_name', - 'message': __( - "Syncing the {eachvar:capitalize} repository"), - 'method': 'Builder.syncLaymanRepository(eachvar)', - 'condition': lambda Get: Get( - 'update.cl_update_other_set') == 'on' - }, - {'name': 'trim_reps', - 'foreach': 'cl_builder_sync_rep', - 'message': __("Cleaning the history of the " - "{eachvar:capitalize} repository"), - 'method': 'Builder.trimRepositories(eachvar)', - 'condition': lambda Get: ( - Get('cl_builder_sync_rep') and - Get('update.cl_update_onedepth_set') == 'on') - }, - {'name': 'regen_cache', - 'foreach': 'cl_builder_sync_overlay_rep', - 'essential': False, - 'method': 'Builder.regenCache(eachvar)', - 'condition': (lambda Get: ( - Get('builder.cl_builder_outdate_set') == 'on' and - Get('update.cl_update_egencache_force') != 'skip' or - Get('update.cl_update_egencache_force') == 'force')) - }, - {'name': 'eix_update', - 'message': __("Updating the eix cache for " - "{cl_builder_eix_repositories}"), - 'method': 'Builder.eixUpdate(cl_builder_repository_name)', - 'condition': (lambda Get: ( - Get('builder.cl_builder_outdate_set') == 'on' and - Get('update.cl_update_eixupdate_force') != 'skip' or - Get('update.cl_update_eixupdate_force') == 'force')) - }, - {'name': 'sync_reps:cleanpkg', - 'message': __( - "Removing obsolete distfiles and binary packages"), - 'method': 'Builder.cleanpkg()', - 'condition': lambda Get: ( - Get('builder.cl_builder_outdate_set') == 'on' and - Get('update.cl_update_cleanpkg_set') == 'on'), - 'essential': False - }, - {'name': 'update_packages_cache', - 'message': __("Update packages index"), - 'method': 'Builder.download_packages(' - 'builder.cl_builder_portage_binhost,' - 'update.cl_update_package_cache)', - 'essential': False, - 'condition': lambda Get: ( - Get('update.cl_update_package_cache') and ( - Get('builder.cl_builder_outdate_set') == 'on' or - Get('update.cl_update_package_cache_set') == 'on')) - }, - # сообщение удачного завершения при обновлении репозиториев - {'name': 'success_syncrep', - 'message': __("Synchronization finished"), - 'depend': (Tasks.success() & Tasks.has_any( - "sync_reps", "sync_other_reps", "emerge_metadata", - "eix_update")), - } - ] - }, + # настроить переменные update для унификации синхронизации репозиториев + {'name': 'prepare_update_vars', + 'method': 'Builder.prepare_update_vars()', + }, + {'name': 'not_use_search', + 'message': __("Will be used base binhost: {builder.cl_builder_binhost_base}"), + 'method': 'Builder.set_base_binhost(' + 'builder.cl_builder_binhost_base)', + 'condition': lambda Get, GetBool: ( + GetBool("builder.cl_builder_binhost_base_set") and + Get("builder.cl_builder_binhost_base")) + }, + ] + get_synchronization_tasks("Builder") + [ {'name': 'reps_synchronization', 'group': __("System configuration"), 'tasks': [ diff --git a/pym/builder/variables/builder.py b/pym/builder/variables/builder.py index 22a57e1..9aad28d 100644 --- a/pym/builder/variables/builder.py +++ b/pym/builder/variables/builder.py @@ -25,6 +25,7 @@ from calculate.lib.utils.dracut import Dracut from calculate.lib.utils.grub import GrubCommand from calculate.lib.utils.portage import getSquashList, isPkgInstalled from calculate.lib.variables.system import RootType +import calculate.update.variables.update as update from .action import Actions import linux as vars_linux import datetime @@ -1817,29 +1818,25 @@ class VariableClBuilderEixRepositories(ReadonlyVariable): class VariableClBuilderBinhostStableSet( - vars_linux.VariableClBuilderLinuxDatavars): + vars_linux.BuilderLinux, update.VariableClUpdateBinhostStableSet): """ Использовать стабильное или нестабильное зеркало """ - type = "bool" - value = "on" - mode = WRITEABLE + variable = "cl_update_binhost_stable_set" - opt = ["--stable"] + def get_fallback(self): + return Variable.On - def init(self): - self.label = _("Use only stable updates") - self.help = _("use only stable updates") - def prepare_vars(self, dvbl): - pass +class VariableClBuilderBinhostStableOptSet( + update.VariableClUpdateBinhostStableOptSet): + """ + Использовать стабильное или нестабильное зеркало + """ + mode = Variable.Writable def get(self): - if self.Get('cl_action') in Actions.WorkAssemble: - dv = self.target_data() - if dv: - return dv.Get('update.cl_update_binhost_stable_opt_set') - return self.Get('update.cl_update_binhost_stable_opt_set') + return self.Get('cl_builder_binhost_stable_set') class VariableClBuilderBinhostScanSet(Variable): diff --git a/pym/builder/variables/linux.py b/pym/builder/variables/linux.py index fed7c00..458f667 100644 --- a/pym/builder/variables/linux.py +++ b/pym/builder/variables/linux.py @@ -48,7 +48,7 @@ class BuilderLinux(ReadonlyVariable): dv = self.Get('cl_builder_linux_datavars') if dv: return dv[self.variable] - except VariableError: + except VariableError as e: pass return self.get_fallback() @@ -77,6 +77,8 @@ class DataVarsBuilderLinux(linux.LinuxDataVars): env.VariableClEnvPath(), env.VariableClEbuildPhase(), update.VariableClUpdateUsetagSet(section="update"), + update.VariableClUpdateGpgForce(section="update"), + update.VariableClUpdateGpgKeys(section="update"), update.VariableClUpdateBinhostBase(section="update"), update.VariableClUpdateBinhostBaseSet(section="update"), update.VariableClUpdateWithBdepsSet(section="update"), @@ -102,8 +104,9 @@ class DataVarsBuilderLinux(linux.LinuxDataVars): update.VariableClUpdateBinhostTimeout(section="update"), update.VariableClUpdateBinhostList(section="update"), update.VariableClUpdateBinhostUnstableList(section="update"), - update.VariableClUpdateBinhostStableSet(section="update"), + update.VariableClUpdateBinhostData(section="update"), update.VariableClUpdateBinhostStableOptSet(section="update"), + update.VariableClUpdateBinhostStableSet(section="update"), update.VariableClUpdateBinhostTimestampPath( section="update"), update.VariableClUpdateBinhostRevisionPath( @@ -153,6 +156,24 @@ class VariableClBuilderBinarySet(ReadonlyVariable): return "on" return "off" +class VariableClBuilderGpgForce(BuilderLinux, update.VariableClUpdateGpgForce): + """ + Использовать GPG проверку + """ + variable = "cl_update_gpg_force" + mode = Variable.Writable + + def get_fallback(self): + return "force" + +class VariableClBuilderBinhost(BuilderLinux, update.VariableClUpdateBinhost): + """ + Принудительное действие с eix-update + """ + variable = "cl_update_binhost" + + def get_fallback(self): + return "" class VariableClBuilderUsetagSet(ReadonlyVariable): """ @@ -179,9 +200,6 @@ class VariableClBuilderLinuxDatavars(ReadonlyVariable): def prepare_vars(self, dvbl): dvbl['cl_update_binhost_recheck_set'] = \ self.Get('update.cl_update_binhost_recheck_set') - dvbl['cl_update_binhost_stable_opt_set'] = \ - self.Get('cl_builder_binhost_stable_set') - dvbl['cl_update_binhost_stable_set'] = "off" if self.Get('cl_action') not in ( Actions.Update, Actions.ChangeProfile) or \ not self.GetBool('cl_builder_binhost_scan_set'): @@ -323,6 +341,48 @@ class VariableOsBuilderLinuxSubname(BuilderLinux): def humanReadable(self): return self.Get() or _("no") +class VariableClBuilderGpgKeys(BuilderLinux): + """ + GPG keys + """ + type = Variable.Types.List + variable = "cl_update_gpg_keys" + + def get_fallback(self): + return [] + +class VariableClBuilderBinhostRevisionPath(BuilderLinux): + """ + Путь до revision файлов + """ + variable = "cl_update_binhost_revision_path" + type = Variable.Types.List + + def get_fallback(self): + value = [ + "grp/ini.env" + ] + +class VariableClBuilderBinhostTimestampPath(BuilderLinux): + """ + Путь до файла timestamp + """ + variable = "cl_update_binhost_timestamp_path" + type = Variable.Types.List + + def get_fallback(self): + value = "timestamp" + + +class VariableClBuilderLastTimestamp(BuilderLinux): + """ + Путь до файла timestamp + """ + variable = "cl_update_last_timestamp" + + def get_fallback(self): + value = "0" + class VariableClBuilderPortdirOverlay(ReadonlyVariable): """ diff --git a/pym/builder/wsdl_builder.py b/pym/builder/wsdl_builder.py index 758277c..9924cae 100644 --- a/pym/builder/wsdl_builder.py +++ b/pym/builder/wsdl_builder.py @@ -177,7 +177,7 @@ class Wsdl(WsdlBase): lambda group: group( _("Update the Build"), normal=('cl_builder_prepared_id', - 'cl_builder_binhost_stable_set', + 'cl_builder_binhost_stable_opt_set', 'update.cl_update_binhost_recheck_set',), expert=( 'update.cl_update_sync_only_set', @@ -187,6 +187,7 @@ class Wsdl(WsdlBase): 'update.cl_update_emergelist_set', 'update.cl_update_check_rep_set', 'update.cl_update_world', + 'builder.cl_builder_gpg_force', 'update.cl_update_egencache_force', 'update.cl_update_eixupdate_force', 'cl_builder_rebuild_changed_set', @@ -201,12 +202,13 @@ class Wsdl(WsdlBase): 'cl_dispatch_conf'), hide=( 'cl_builder_prepared_id', - 'cl_builder_binhost_stable_set', + 'cl_builder_binhost_stable_opt_set', 'update.cl_update_binhost_recheck_set', 'update.cl_update_sync_only_set', 'update.cl_update_other_set', 'update.cl_update_pretend_set', 'cl_builder_sync_rep', + 'builder.cl_builder_gpg_force', 'update.cl_update_emergelist_set', 'update.cl_update_check_rep_set', 'update.cl_update_world', @@ -225,7 +227,7 @@ class Wsdl(WsdlBase): 'cl_dispatch_conf'), brief=( 'cl_builder_prepared_id', - 'cl_builder_binhost_stable_set', + 'cl_builder_binhost_stable_opt_set', 'update.cl_update_binhost_recheck_set', 'update.cl_update_sync_only_set', 'update.cl_update_other_set', @@ -234,6 +236,7 @@ class Wsdl(WsdlBase): 'update.cl_update_emergelist_set', 'update.cl_update_check_rep_set', 'update.cl_update_world', + 'builder.cl_builder_gpg_force', 'update.cl_update_egencache_force', 'update.cl_update_eixupdate_force', 'cl_builder_rebuild_changed_set',