Добавлена проверка подписи Packages

pym/builder/builder.py

@@ -25,7 +25,7 @@ import json
 from calculate.core.server.gen_pid import search_worked_process2
 from calculate.core.setup_package import ChainProgressTemplate
 from calculate.lib.cl_template import templateFunction, SystemIni
-from calculate.lib.datavars import DataVars, Variable
+from calculate.lib.datavars import DataVars, Variable, DataVarsError
 from calculate.lib.utils.colortext import (TextState, get_color_print)
 from calculate.lib.utils.common import CmdlineParams, getTupleVersion
 from calculate.lib.utils.portage import (BinaryPackage, get_binary_file,
@@ -60,7 +60,7 @@ from calculate.update.emerge_parser import (EmergeParser,
 from calculate.lib.cl_log import log
 from calculate.update.update_tasks import EmergeMark
 from .build_storage import Build
-from calculate.update.update import Update
+from calculate.update.update import Update, variable_module
 from calculate.install.distr import (Distributive, IsoDistributive,
                                      DistributiveError, ContainerDistributive)
 import shutil
@@ -102,6 +102,7 @@ class Builder(Update):
         self.pkgnum = None
         self.pkgnummax = None
         self.world_data = ""
+        self.binhosts_data = None
 
     def mount_target(self, target):
         dir_distro = target.convertToDirectory()
@@ -593,6 +594,10 @@ class Builder(Update):
         with self.clVars.useDefaultModule("update"):
             return super(Builder, self).regenCache(repname)
 
+    def prepare_gpg(self):
+        with self.clVars.useDefaultModule("update"):
+            return super(Builder, self).prepare_gpg()
+
     def getGit(self):
         chroot_path = self.clVars.Get('builder.cl_builder_path')
         sshkey = pathJoin(chroot_path,
@@ -624,6 +629,7 @@ class Builder(Update):
         makeDirectory(logpath)
         return True
 
+    @variable_module("builder")
     def _get_log_file(self):
         logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'),
                                    self.clVars.Get('cl_task_name'))
@@ -1107,12 +1113,13 @@ class Builder(Update):
             self._startEmerging(emerge)
         return True
 
-    def update_layman(self, builder_path):
+    def update_layman(self):
         """
         Обновить базу layman
         :param builder_path:
         :return:
         """
+        builder_path = self.clVars.Get("cl_builder_path")
         cmd = "/usr/bin/layman"
         cmd_path = self.get_prog_path(cmd)
         logfile = self._get_log_file()
@@ -1310,23 +1317,57 @@ class Builder(Update):
             raise BuilderError(str(e))
         return True
 
-    def raiseOutdate(self):
+    def prepare_update_vars(self):
         """
-        Установить флаг данные о репозиториях устарели (необходим для выполнения
-        eix-update и прочих команд обновляющих кэш
-        :return:
+        Заместить значения update переменных одноимёнными из builder
         """
-        self.clVars.Set('cl_builder_outdate_set', 'on', force=True)
+        vars_map = {
+            #Применить значение переменной для выбора веток репозиториев
+            #при обновлении
+            'update.cl_update_branch_name': 'builder.cl_builder_branch_name',
 
-    def apply_branch_variables(self):
-        """
-        Применить значение переменной для выбора веток репозиториев
-        при обновлении
-        """
-        self.clVars.Set('update.cl_update_branch_name',
-                        self.clVars.Get('builder.cl_builder_branch_name'))
+            'update.cl_update_gpg_force': 'builder.cl_builder_gpg_force',
+            'update.cl_update_gpg_keys': 'builder.cl_builder_gpg_keys',
+            'update.cl_update_binhost_revision_path': 
+                'builder.cl_builder_binhost_revision_path',
+            'update.cl_update_binhost_timestamp_path': 
+                'builder.cl_builder_binhost_timestamp_path',
+            'update.cl_update_last_timestamp': 
+                'builder.cl_builder_last_timestamp',
+
+            'update.cl_update_binhost_stable_set':
+                'builder.cl_builder_binhost_stable_set',
+            'update.cl_update_binhost_stable_opt_set':
+                'builder.cl_builder_binhost_stable_opt_set',
+
+            'update.cl_update_binhost':
+                'builder.cl_builder_binhost',
+
+            'update.cl_update_sync_rep': 'builder.cl_builder_sync_rep',
+            'update.cl_update_other_rep_name': 'builder.cl_builder_other_rep_name',
+            'update.cl_update_usetag_set': 'builder.cl_builder_usetag_set',
+
+            'update.cl_update_sync_overlay_rep': 'builder.cl_builder_sync_overlay_rep',
+            'update.cl_repository_name': 'builder.cl_builder_repository_name'
+        }
+        try:
+            for k,v in vars_map.items():
+                self.clVars.Set(k, self.clVars.Get(v), force=True)
+        except DataVarsError as e:
+            error = BuilderError(_("Failed to prepare variables for synchronization"))
+            error.addon = e
+            raise error
         return True
 
+    @variable_module("builder")
+    def _get_binhost_logger(self):
+        logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'),
+                                   "binhost-scan.log")
+        mainlog = self.clVars.Get('core.cl_log_path')
+        return log("binhost-scan.log",
+                   filename=path.join(mainlog, logname),
+                   formatter="%(message)s")
+
     def isohybrid(self, image_file):
         """
         Преобразовать ISO образ в гибридный
@@ -1390,6 +1431,9 @@ class Builder(Update):
             return False
         return True
 
+    def get_arch_machine(self):
+        return self.clVars.Get("builder.os_builder_arch_machine")
+
     @property
     def chroot_process(self):
         chroot_cmd = getProgPath("/usr/bin/chroot")
@@ -2033,6 +2077,8 @@ class Builder(Update):
         """
         self.clVars.Write('cl_update_binhost', binhost)
         self.clVars.Set('cl_update_package_cache_set', 'on')
+        self.clVars.Set('cl_update_binhost_list', [binhost], force=True)
+        self.clVars.Set('cl_update_binhost_unstable_list', [binhost], force=True)
         return True
 
     def index_pkgdir(self, pkgdir, trunkdir, stabledir):
@@ -2083,3 +2129,13 @@ class Builder(Update):
             raise BuilderError(
                 _("Failed to merge prepared binaries: %s") % str(e))
         return True
+
+    @variable_module("builder")
+    def update_binhost_list(self):
+        self.invalidateVariables('cl_builder_linux_datavars')
+        self.prepare_update_vars()
+        return super(Builder,self).update_binhost_list(
+                     self.clVars.Get('cl_builder_linux_datavars'))
+
+    def is_update_action(self, action):
+        return action == Actions.Update

pym/builder/utils/cl_builder_update.py

@@ -29,6 +29,7 @@ from calculate.lib.utils.portage import (EmergeLogNamedTask,
                                          PackageList, EmergeLog, isPkgInstalled)
 from calculate.install.distr import DistributiveError
 from calculate.update.update_tasks import EmergeMark
+from calculate.update.utils.cl_update import get_synchronization_tasks
 
 _ = lambda x: x
 setLocalTranslate('cl_builder3', sys.modules[__name__])
@@ -112,128 +113,19 @@ class ClBuilderUpdateAction(Action):
                  'method': 'Builder.reconfigureProfileVars('
                            'cl_builder_linux_datavars,cl_builder_path)'
                  },
-                {'name': 'apply_branch_vars',
-                 'method': 'Builder.apply_branch_variables()'
-                 },
-                {'name': 'reps_synchronization',
-                 'group': __("Repositories synchronization"),
-                 'tasks': [
-                     # запасная синхронизация, в ходе которой ветки
-                     #  обновляются до master
-                     {'name': 'sync_reps_fallback',
-                      'foreach': 'cl_builder_sync_rep',
-                      'message':
-                          __("Fallback syncing the {eachvar:capitalize} "
-                             "repository"),
-                      'method': 'Builder.syncRepositories(eachvar,True)',
-                      'condition': lambda Get: (
-                          Get('cl_builder_usetag_set') == "on" and
-                          not Get('update.cl_update_binhost_data')[0])
-                      },
-                     # обновление переменных информации из binhost
-                     {'name': 'invalidate_vars',
-                      'method':
-                          'Builder.invalidateVariables('
-                          '"cl_builder_linux_datavars")',
-                      'condition': lambda Get: (
-                          Get('cl_builder_usetag_set') == "on" and
-                          not Get('update.cl_update_binhost_data')[0])
-                      },
-                     {'name': 'binhost_unavailable',
-                      'error': __("Update server is unavailable"),
-                      'condition': lambda Get: (
-                          Get('cl_builder_usetag_set') == "on" and
-                          not Get('update.cl_update_binhost_data')[0])
-                      },
-                     {'name': 'sync_reps',
-                      'foreach': 'cl_builder_sync_rep',
-                      'message': __("Checking {eachvar:capitalize} updates"),
-                      'method': 'Builder.syncRepositories(eachvar)',
-                      'condition': lambda Get: Get('cl_builder_sync_rep')
-                      },
-                     {'name': 'check_binhost',
-                      'method': 'Builder.check_binhost(True)',
-                      'condition': lambda Get: Get(
-                          'cl_builder_usetag_set') == "on"
-                      },
-                     {'name': 'use_base_binhost',
-                      'method': 'Builder.set_base_binhost('
-                                'builder.cl_builder_binhost_base)',
-                      'condition': lambda Get, GetBool: (
-                            GetBool("builder.cl_builder_binhost_base_set") and
-                            Get("builder.cl_builder_binhost_base"))
-                      },
-                     {'name': 'update_layman',
-                      'message': __("Layman cache update"),
-                      'method': "Builder.update_layman(cl_builder_path)",
-                      'condition': lambda Get: isPkgInstalled(
-                          "app-portage/layman", prefix=Get('cl_chroot_path')),
-                      'essential': False,
-                      },
-                     {'name': 'sync_other_reps',
-                      'foreach': 'builder.cl_builder_other_rep_name',
-                      'message': __(
-                          "Syncing the {eachvar:capitalize} repository"),
-                      'method': 'Builder.syncLaymanRepository(eachvar)',
-                      'condition': lambda Get: Get(
-                          'update.cl_update_other_set') == 'on'
-                      },
-                     {'name': 'trim_reps',
-                      'foreach': 'cl_builder_sync_rep',
-                      'message': __("Cleaning the history of the "
-                                    "{eachvar:capitalize} repository"),
-                      'method': 'Builder.trimRepositories(eachvar)',
-                      'condition': lambda Get: (
-                          Get('cl_builder_sync_rep') and
-                          Get('update.cl_update_onedepth_set') == 'on')
-                      },
-                     {'name': 'regen_cache',
-                      'foreach': 'cl_builder_sync_overlay_rep',
-                      'essential': False,
-                      'method': 'Builder.regenCache(eachvar)',
-                      'condition': (lambda Get: (
-                          Get('builder.cl_builder_outdate_set') == 'on' and
-                          Get('update.cl_update_egencache_force') != 'skip' or
-                          Get('update.cl_update_egencache_force') == 'force'))
-                      },
-                     {'name': 'eix_update',
-                      'message': __("Updating the eix cache for "
-                                    "{cl_builder_eix_repositories}"),
-                      'method': 'Builder.eixUpdate(cl_builder_repository_name)',
-                      'condition': (lambda Get: (
-                          Get('builder.cl_builder_outdate_set') == 'on' and
-                          Get('update.cl_update_eixupdate_force') != 'skip' or
-                          Get('update.cl_update_eixupdate_force') == 'force'))
-                      },
-                     {'name': 'sync_reps:cleanpkg',
-                      'message': __(
-                          "Removing obsolete distfiles and binary packages"),
-                      'method': 'Builder.cleanpkg()',
-                      'condition': lambda Get: (
-                          Get('builder.cl_builder_outdate_set') == 'on' and
-                          Get('update.cl_update_cleanpkg_set') == 'on'),
-                      'essential': False
-                      },
-                     {'name': 'update_packages_cache',
-                      'message': __("Update packages index"),
-                      'method': 'Builder.download_packages('
-                                'builder.cl_builder_portage_binhost,'
-                                'update.cl_update_package_cache)',
-                      'essential': False,
-                      'condition': lambda Get: (
-                          Get('update.cl_update_package_cache') and (
-                              Get('builder.cl_builder_outdate_set') == 'on' or
-                              Get('update.cl_update_package_cache_set') == 'on'))
-                      },
-                     # сообщение удачного завершения при обновлении репозиториев
-                     {'name': 'success_syncrep',
-                      'message': __("Synchronization finished"),
-                      'depend': (Tasks.success() & Tasks.has_any(
-                          "sync_reps", "sync_other_reps", "emerge_metadata",
-                          "eix_update")),
-                      }
-                 ]
-                 },
+                # настроить переменные update для унификации синхронизации репозиториев
+                {'name': 'prepare_update_vars',
+                 'method': 'Builder.prepare_update_vars()',
+                },
+                {'name': 'not_use_search',
+                 'message': __("Will be used base binhost: {builder.cl_builder_binhost_base}"),
+                 'method': 'Builder.set_base_binhost('
+                           'builder.cl_builder_binhost_base)',
+                 'condition': lambda Get, GetBool: (
+                        GetBool("builder.cl_builder_binhost_base_set") and
+                        Get("builder.cl_builder_binhost_base"))
+                },
+                ] + get_synchronization_tasks("Builder") + [
                 {'name': 'reps_synchronization',
                  'group': __("System configuration"),
                  'tasks': [

pym/builder/variables/builder.py

@@ -25,6 +25,7 @@ from calculate.lib.utils.dracut import Dracut
 from calculate.lib.utils.grub import GrubCommand
 from calculate.lib.utils.portage import getSquashList, isPkgInstalled
 from calculate.lib.variables.system import RootType
+import calculate.update.variables.update as update
 from .action import Actions
 import linux as vars_linux
 import datetime
@@ -1817,29 +1818,25 @@ class VariableClBuilderEixRepositories(ReadonlyVariable):
 
 
 class VariableClBuilderBinhostStableSet(
-    vars_linux.VariableClBuilderLinuxDatavars):
+    vars_linux.BuilderLinux, update.VariableClUpdateBinhostStableSet):
     """
     Использовать стабильное или нестабильное зеркало
     """
-    type = "bool"
-    value = "on"
-    mode = WRITEABLE
+    variable = "cl_update_binhost_stable_set"
 
-    opt = ["--stable"]
+    def get_fallback(self):
+        return Variable.On
 
-    def init(self):
-        self.label = _("Use only stable updates")
-        self.help = _("use only stable updates")
 
-    def prepare_vars(self, dvbl):
-        pass
+class VariableClBuilderBinhostStableOptSet(
+    update.VariableClUpdateBinhostStableOptSet):
+    """
+    Использовать стабильное или нестабильное зеркало
+    """
+    mode = Variable.Writable
 
     def get(self):
-        if self.Get('cl_action') in Actions.WorkAssemble:
-            dv = self.target_data()
-            if dv:
-                return dv.Get('update.cl_update_binhost_stable_opt_set')
-        return self.Get('update.cl_update_binhost_stable_opt_set')
+        return self.Get('cl_builder_binhost_stable_set')
 
 
 class VariableClBuilderBinhostScanSet(Variable):

pym/builder/variables/linux.py

@@ -48,7 +48,7 @@ class BuilderLinux(ReadonlyVariable):
             dv = self.Get('cl_builder_linux_datavars')
             if dv:
                 return dv[self.variable]
-        except VariableError:
+        except VariableError as e:
             pass
         return self.get_fallback()
 
@@ -77,6 +77,8 @@ class DataVarsBuilderLinux(linux.LinuxDataVars):
                     env.VariableClEnvPath(),
                     env.VariableClEbuildPhase(),
                     update.VariableClUpdateUsetagSet(section="update"),
+                    update.VariableClUpdateGpgForce(section="update"),
+                    update.VariableClUpdateGpgKeys(section="update"),
                     update.VariableClUpdateBinhostBase(section="update"),
                     update.VariableClUpdateBinhostBaseSet(section="update"),
                     update.VariableClUpdateWithBdepsSet(section="update"),
@@ -102,8 +104,9 @@ class DataVarsBuilderLinux(linux.LinuxDataVars):
                     update.VariableClUpdateBinhostTimeout(section="update"),
                     update.VariableClUpdateBinhostList(section="update"),
                     update.VariableClUpdateBinhostUnstableList(section="update"),
-                    update.VariableClUpdateBinhostStableSet(section="update"),
+                    update.VariableClUpdateBinhostData(section="update"),
                     update.VariableClUpdateBinhostStableOptSet(section="update"),
+                    update.VariableClUpdateBinhostStableSet(section="update"),
                     update.VariableClUpdateBinhostTimestampPath(
                         section="update"),
                     update.VariableClUpdateBinhostRevisionPath(
@@ -153,6 +156,24 @@ class VariableClBuilderBinarySet(ReadonlyVariable):
                 return "on"
         return "off"
 
+class VariableClBuilderGpgForce(BuilderLinux, update.VariableClUpdateGpgForce):
+    """
+    Использовать GPG проверку
+    """
+    variable = "cl_update_gpg_force"
+    mode = Variable.Writable
+
+    def get_fallback(self):
+        return "force"
+
+class VariableClBuilderBinhost(BuilderLinux, update.VariableClUpdateBinhost):
+    """
+    Принудительное действие с eix-update
+    """
+    variable = "cl_update_binhost"
+
+    def get_fallback(self):
+        return ""
 
 class VariableClBuilderUsetagSet(ReadonlyVariable):
     """
@@ -179,9 +200,6 @@ class VariableClBuilderLinuxDatavars(ReadonlyVariable):
     def prepare_vars(self, dvbl):
         dvbl['cl_update_binhost_recheck_set'] = \
             self.Get('update.cl_update_binhost_recheck_set')
-        dvbl['cl_update_binhost_stable_opt_set'] = \
-            self.Get('cl_builder_binhost_stable_set')
-        dvbl['cl_update_binhost_stable_set'] = "off"
         if self.Get('cl_action') not in (
                 Actions.Update, Actions.ChangeProfile) or \
                 not self.GetBool('cl_builder_binhost_scan_set'):
@@ -323,6 +341,48 @@ class VariableOsBuilderLinuxSubname(BuilderLinux):
     def humanReadable(self):
         return self.Get() or _("no")
 
+class VariableClBuilderGpgKeys(BuilderLinux):
+    """
+    GPG keys
+    """
+    type = Variable.Types.List
+    variable = "cl_update_gpg_keys"
+
+    def get_fallback(self):
+        return []
+
+class VariableClBuilderBinhostRevisionPath(BuilderLinux):
+    """
+    Путь до revision файлов
+    """
+    variable = "cl_update_binhost_revision_path"
+    type = Variable.Types.List
+
+    def get_fallback(self):
+        value = [
+            "grp/ini.env"
+    ]
+
+class VariableClBuilderBinhostTimestampPath(BuilderLinux):
+    """
+    Путь до файла timestamp
+    """
+    variable = "cl_update_binhost_timestamp_path"
+    type = Variable.Types.List
+
+    def get_fallback(self):
+        value = "timestamp"
+
+
+class VariableClBuilderLastTimestamp(BuilderLinux):
+    """
+    Путь до файла timestamp
+    """
+    variable = "cl_update_last_timestamp"
+
+    def get_fallback(self):
+        value = "0"
+
 
 class VariableClBuilderPortdirOverlay(ReadonlyVariable):
     """

pym/builder/wsdl_builder.py

@@ -177,7 +177,7 @@ class Wsdl(WsdlBase):
                 lambda group: group(
                     _("Update the Build"),
                     normal=('cl_builder_prepared_id',
-                            'cl_builder_binhost_stable_set',
+                            'cl_builder_binhost_stable_opt_set',
                             'update.cl_update_binhost_recheck_set',),
                     expert=(
                         'update.cl_update_sync_only_set',
@@ -187,6 +187,7 @@ class Wsdl(WsdlBase):
                         'update.cl_update_emergelist_set',
                         'update.cl_update_check_rep_set',
                         'update.cl_update_world',
+                        'builder.cl_builder_gpg_force',
                         'update.cl_update_egencache_force',
                         'update.cl_update_eixupdate_force',
                         'cl_builder_rebuild_changed_set',
@@ -201,12 +202,13 @@ class Wsdl(WsdlBase):
                         'cl_dispatch_conf'),
                     hide=(
                         'cl_builder_prepared_id',
-                        'cl_builder_binhost_stable_set',
+                        'cl_builder_binhost_stable_opt_set',
                         'update.cl_update_binhost_recheck_set',
                         'update.cl_update_sync_only_set',
                         'update.cl_update_other_set',
                         'update.cl_update_pretend_set',
                         'cl_builder_sync_rep',
+                        'builder.cl_builder_gpg_force',
                         'update.cl_update_emergelist_set',
                         'update.cl_update_check_rep_set',
                         'update.cl_update_world',
@@ -225,7 +227,7 @@ class Wsdl(WsdlBase):
                         'cl_dispatch_conf'),
                     brief=(
                         'cl_builder_prepared_id',
-                        'cl_builder_binhost_stable_set',
+                        'cl_builder_binhost_stable_opt_set',
                         'update.cl_update_binhost_recheck_set',
                         'update.cl_update_sync_only_set',
                         'update.cl_update_other_set',
@@ -234,6 +236,7 @@ class Wsdl(WsdlBase):
                         'update.cl_update_emergelist_set',
                         'update.cl_update_check_rep_set',
                         'update.cl_update_world',
+                        'builder.cl_builder_gpg_force',
                         'update.cl_update_egencache_force',
                         'update.cl_update_eixupdate_force',
                         'cl_builder_rebuild_changed_set',