Добавлена проверка подписи Packages

pym/update/update.py

@@ -24,7 +24,7 @@ from calculate.core.server.gen_pid import search_worked_process
 from calculate.core.setup_cache import Cache as SetupCache
 from calculate.core.server.func import MethodsInterface
 from calculate.lib.cl_template import SystemIni, LayeredIni
-from calculate.lib.datavars import DataVarsError, VariableError
+from calculate.lib.datavars import DataVarsError, VariableError, Variable
 
 from calculate.lib.utils.tools import AddonError
 from calculate.lib.utils.colortext.palette import TextState
@@ -32,12 +32,16 @@ from calculate.lib.utils.colortext import get_color_print
 from calculate.update.emerge_parser import RevdepPercentBlock
 from calculate.update.datavars import DataVarsUpdate
 from calculate.update.update_info import UpdateInfo
-from calculate.lib.utils.binhosts import fetch_packages
+from calculate.lib.utils.binhosts import (Binhosts, BinhostSignError,
+    BinhostError, PackagesIndex, DAYS)
+from calculate.lib.utils.gpg import GPG, GPGError
 from calculate.lib.cl_log import log
+import hashlib
 import re
 import shutil
 from collections import MutableSet
 from contextlib import contextmanager
+import tempfile
 
 from calculate.lib.utils.git import Git, GitError, MTimeKeeper, NotGitError
 from calculate.lib.utils.portage import (Layman, EmergeLog,
@@ -52,7 +56,7 @@ Colors = TextState.Colors
 from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
                                        PercentProgress, process, getRunCommands,
                                        readFile, listDirectory, pathJoin,
-                                       find, FindFileType,
+                                       find, FindFileType,quite_unlink,
                                        writeFile, makeDirectory)
 import emerge_parser
 import logging
@@ -151,6 +155,8 @@ class Update(MethodsInterface):
         self.refresh_binhost = False
         self.pkgnum = None
         self.pkgnummax = None
+        self.gpgdata_md5 = []
+        self.gpg_changed = False
 
     def get_prog_path(self, program_name):
         return getProgPath(program_name)
@@ -217,7 +223,7 @@ class Update(MethodsInterface):
                         status = git.getStatusInfo(rpath)
                         if not status or status['files']:
                             need_update = True
-                except GitError:
+                except GitError as e:
                     need_update = True
                 if need_update:
                     if not notask:
@@ -236,6 +242,8 @@ class Update(MethodsInterface):
                 dv.Set('cl_update_outdate_set', 'on', force=True)
         finally:
             self.unstash_cache(rpath, name)
+        # TODO: debug1
+        #dv.Set('cl_update_outdate_set', 'on', force=True)
         return True
 
     def raiseOutdate(self):
@@ -1213,28 +1221,23 @@ class Update(MethodsInterface):
         cache.update(force=True)
         return True
 
-    def check_binhost(self, write_binhost=True):
+    def get_bin_cache_filename(self):
+        return pathJoin(self.clVars.Get('cl_chroot_path'),
+                        LayeredIni.IniPath.Grp)
+
+    def update_local_info_binhost(self, write_binhost=True):
         """
         Проверить, что доступен хотя бы один из binhost'ов
         :return:
         """
         hosts = self.clVars.Get("update.cl_update_binhost_host")
         datas = self.clVars.Get("update.cl_update_binhost_revisions")
-        bin_cache_fn = pathJoin(self.clVars.Get('cl_chroot_path'),
-                                LayeredIni.IniPath.Grp)
         if not hosts:
-            self.clVars.Delete('cl_update_binhost', location="system")
+            self.delete_binhost()
-            try:
-                if path.exists(bin_cache_fn):
-                    os.unlink(bin_cache_fn)
-            except OSError:
-                raise UpdateError(
-                    _("Failed to remove cached ini.env of binary repository"))
-
             raise UpdateError(_("Failed to find the server with "
                                 "appropriate updates"))
         else:
-            with writeFile(bin_cache_fn) as f:
+            with writeFile(self.get_bin_cache_filename()) as f:
                 f.write(datas[0].strip()+"\n")
         if write_binhost:
             if hosts[0] != self.clVars.Get('update.cl_update_binhost'):
@@ -1275,33 +1278,53 @@ class Update(MethodsInterface):
                               self.clVars.Get('cl_update_binhost'))
         return True
 
+    def delete_binhost(self):
+        self.clVars.Delete('cl_update_binhost', location="system")
+        try:
+            bin_cache_fn = self.get_bin_cache_filename()
+            if path.exists(bin_cache_fn):
+                os.unlink(bin_cache_fn)
+        except OSError:
+            raise UpdateError(
+                _("Failed to remove cached ini.env of binary repository"))
+        try:
+            for varname in ('cl_update_package_cache', 'cl_update_package_cache_sign'):
+                fn = self.clVars.Get(varname)
+                if path.exists(fn):
+                    os.unlink(fn)
+        except OSError:
+            raise UpdateError(
+                _("Failed to remove cached Package index"))
+        return True
+
     def update_binhost_list(self):
         """
         Обновить список binhost'ов после обновления до master веток
         :return:
         """
-        dv = DataVarsUpdate()
+        changes = False
         try:
+            dv = DataVarsUpdate()
             dv.importUpdate()
             dv.flIniFile()
-            changes = False
             for varname in ('update.cl_update_binhost_list',
                             'update.cl_update_binhost_unstable_list',
                             'update.cl_update_binhost_timestamp_path',
+                            'update.cl_update_gpg_keys',
                             'cl_update_binhost_revision_path'):
                 new_value = dv.Get(varname)
                 old_value = self.clVars.Get(varname)
                 if new_value != old_value:
                     changes = True
-                self.clVars.Set(varname, new_value, force=True)
+                    self.clVars.Set(varname, new_value, force=True)
-            if not changes:
-                self.clVars.Delete('cl_update_binhost', location="system")
-                raise UpdateError(_("Failed to find the server with "
-                                    "appropriate updates"))
         except DataVarsError:
-            self.clVars.Delete('cl_update_binhost', location="system")
+            raise UpdateError(_("Failed to get values for binhost search"))
-            raise UpdateError(_("Failed to find the server with "
+
-                                "appropriate updates"))
+        if self.is_gpg_changed():
+            self.prepare_gpg()
+        elif not changes:
+            return False
+        self.create_binhost_data()
         return True
 
     def drop_binhosts(self, dv):
@@ -1316,8 +1339,271 @@ class Update(MethodsInterface):
         dv.Invalidate('update.cl_update_rep_rev')
         return True
 
-    def download_packages(self, url_binhost, packages_fn):
+    def is_gpg_changed(self):
-        fetch_packages(url_binhost, packages_fn)
+        """
+        Проверить изменились ли открытые ключи
+        """
+        gpg_force = self.clVars.Get('cl_update_gpg_force')
+        if gpg_force == "skip":
+            return False
+        gpg_keys = self.clVars.Get('cl_update_gpg_keys')
+        gpgdata = ""
+        for keyfn in gpg_keys:
+            if path.exists(keyfn):
+                gpgdata += readFile(keyfn)
+        new_gpgdata_md5 = hashlib.md5(gpgdata).hexdigest()
+        return all(new_gpgdata_md5 != x for x in self.gpgdata_md5)
+
+    def prepare_gpg(self):
+        """
+        Получить объект для проверки подписи, либо получить заглушку
+        """
+        gpg_force = self.clVars.Get('cl_update_gpg_force')
+        gpg_keys = self.clVars.Get('cl_update_gpg_keys')
+        if gpg_force == "skip":
+            return True
+        gpg = GPG(tempfile.mkdtemp(dir="/var/calculate/tmp/update",
+                                   prefix="gpg-"))
+        gpgdata = ""
+        for keyfn in gpg_keys:
+            if path.exists(keyfn):
+                try:
+                    key = readFile(keyfn)
+                    gpgdata += key
+                    gpg.import_key(key)
+                except GPGError as e:
+                    self.printWARNING(_("Failed to load public keys from '%s' "
+                        "for signature checking") % keyfn)
+        self.gpgdata_md5.append(hashlib.md5(gpgdata).hexdigest())
+        if len(self.gpgdata_md5) > 1:
+            self.gpg_changed = True
+        if not gpg.count_public():
+            if gpg_force == "force":
+                raise UpdateError(_("Public keys for Packages signature checking not found"))
+            else:
+                return True
+        oldgpg = self.clVars.Get('update.cl_update_gpg')
+        if oldgpg:
+            oldgpg.close()
+        self.clVars.Set('update.cl_update_gpg', gpg, force=True)
+        return True
+
+    def download_packages(self, url_binhost, packages_fn, packages_sign_fn, gpg):
+        quite_unlink(packages_fn)
+        orig_packages = Binhosts.fetch_packages(url_binhost)
+        try:
+            with writeFile(packages_fn) as f:
+                pi = PackagesIndex(orig_packages)
+                pi["TTL"] = str(30 * DAYS)
+                pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time()))
+                pi.write(f)
+            with writeFile("/tmp/Packages.org") as f:
+                f.write(orig_packages)
+                #pi = PackagesIndex(orig_packages)
+                #pi.write(f)
+        except (OSError, IOError):
+            raise UpdateError(_("Failed to save Packages"))
+        self.endTask(True)
+        self.startTask(_("Check packages index signature"))
+        if not gpg:
+            self.endTask("skip")
+            self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True)
+            return True
+        try:
+            Binhosts.check_packages_signature(
+                url_binhost, orig_packages, gpg)
+            with writeFile(packages_sign_fn) as f:
+                f.write(Binhosts.fetch_packages_sign(url_binhost))
+        except BinhostSignError:
+            for fn in (packages_fn, packages_sign_fn):
+                if path.exists(fn):
+                    try:
+                        os.unlink(fn)
+                    except OSError:
+                        pass
+            self.clVars.Set("update.cl_update_bad_sign_set", Variable.On)
+            self.clVars.Set('update.cl_update_binhost_recheck_set', Variable.On)
+            self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True)
+            raise
+        return True
+
+    class Reason(object):
+        WrongBinhost = "wrong_binhost"
+        Outdated = "outdated"
+        Updating = "updating"
+        BadEnv = "badenv"
+        EnvNotFound = "noenv"
+        UnknownError = "unknown"
+        BadSign = "badsign"
+        Skip = "skip"
+        SkipSlower = "skipslower"
+        Success = "success"
+
+        @staticmethod
+        def humanReadable(reason):
+            return {
+                Update.Reason.WrongBinhost: "FAILED (Wrong binhost)",
+                Update.Reason.Outdated: "OUTDATED",
+                Update.Reason.Updating: "UPDATING",
+                Update.Reason.BadEnv: "FAILED (Bad env)",
+                Update.Reason.EnvNotFound: "FAILED (Env not found)",
+                Update.Reason.UnknownError: "FAILED (Unknown error)",
+                Update.Reason.BadSign: "FAILED (Bad sign)",
+                Update.Reason.Skip: "SKIP",
+                Update.Reason.SkipSlower: "",
+                Update.Reason.Success: ""
+            }.get(reason,reason)
+
+    def _get_binhost_logger(self):
+        return log("binhost-scan.log",
+                   filename=pathJoin(
+                       self.clVars.Get('cl_chroot_path'),
+                       "/var/log/calculate/binhost-scan.log"),
+                   formatter="%(message)s")
+
+    def create_binhost_data(self):
+        dv = self.clVars
+        last_ts = dv.Get('cl_update_last_timestamp')
+        if dv.GetBool('cl_update_binhost_stable_opt_set'):
+            binhost_list = dv.Get('cl_update_binhost_list')
+        else:
+            binhost_list = dv.Get('cl_update_binhost_unstable_list')
+        self.binhosts_data = Binhosts(
+            dv.GetInteger('cl_update_binhost_timeout'),
+            dv.Get('cl_update_binhost_revision_path'),
+            dv.Get('cl_update_binhost_timestamp_path'),
+            last_ts, binhost_list,
+            dv.Get('os_arch_machine'),
+            gpg=dv.Get('cl_update_gpg'))
         return True
 
+    def _search_best_binhost(self, binhosts_data, stabilization):
+        if not self.clVars.Get('cl_ebuild_phase'):
+            logger = self._get_binhost_logger()
+        if logger:
+            logger.info(
+                "Started scan on: {date}, current timestamp: {ts}".format(
+                date=time.ctime(), ts=binhosts_data.last_ts))
+        retval = []
+        skip_check_status = False
+        for binhost in sorted(binhosts_data.get_binhosts(), reverse=True):
+            host = binhost.host
+            if not binhost.valid:
+                reason = self.Reason.WrongBinhost 
+            elif binhost.outdated:
+                reason = self.Reason.Outdated 
+            elif not skip_check_status:
+                status = binhost.status
+                if status is not binhosts_data.BinhostStatus.Success:
+                    errors = {
+                        binhosts_data.BinhostStatus.Updating: self.Reason.Updating,
+                        binhosts_data.BinhostStatus.BadEnv: self.Reason.BadEnv,
+                        binhosts_data.BinhostStatus.EnvNotFound: self.Reason.EnvNotFound 
+                    }
+                    reason = errors.get(status, self.Reason.UnknownError)
+                elif binhost.bad_sign:
+                    reason = self.Reason.BadSign
+                else:
+                    # SUCCESS
+                    if not binhost.downgraded or stabilization:
+                        host = "-> %s" % host
+                        reason = self.Reason.Success
+                    else:
+                        reason = self.Reason.Skip
+            elif binhost.downgraded:
+                reason = self.Reason.Skip
+            else:
+                reason = self.Reason.SkipSlower
+
+            if reason == self.Reason.Success:
+                retval.append([binhost.host, binhost.data,
+                            str(binhost.timestamp),
+                            str(binhost.duration)])
+                skip_check_status = True
+            logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format(
+                host=host, speed=float(binhost.duration) / 1000.0,
+                timestamp=binhost.timestamp,
+                reason=Update.Reason.humanReadable(reason)))
+        if not retval:
+            raise UpdateError(_("Failed to find the server with appropriate updates"))
+        return retval
+        
+    def check_current_binhost(self, binhost_url):
+        """
+        Проверка текущего сервера обновлений на валидность
+        """
+        if not binhost_url in self.binhosts_data.binhost_list:
+            raise UpdateError(_("Current binhost is absent in list of update servers"))
+        binhost = self.binhosts_data.get_binhost(binhost_url)
+
+        if binhost.valid and not binhost.outdated and not binhost.downgraded:
+            if binhost.status == self.binhosts_data.BinhostStatus.Success:
+                self.clVars.Set('update.cl_update_binhost_data',
+                        [[binhost.host, binhost.data,
+                         str(binhost.timestamp),
+                         str(binhost.duration)]],
+                         force=True)
+                self.endTask()
+        else:
+            if not binhost.valid:
+                raise UpdateError(
+                    _("Current binhost {} is not valid").format(binhost_url))
+            elif binhost.outdated:
+                raise UpdateError(
+                    _("Current binhost {} is outdated").format(binhost_url))
+            elif binhost.downgraded:
+                raise UpdateError(
+                    _("Current binhost {} was downgraded").format(binhost_url))
+        if self.binhosts_data.gpg:
+            packages_fn = self.clVars.Get('cl_update_package_cache')
+            packages_sign_fn = self.clVars.Get('cl_update_package_cache_sign')
+            if path.exists(packages_fn) and path.exists(packages_sign_fn):
+                packages_sign = readFile(packages_sign_fn)
+                pi = PackagesIndex(readFile(packages_fn))
+                pi.clean()
+                try:
+                    Binhosts.check_packages_signature(
+                        None, pi.get_value(), self.binhosts_data.gpg,
+                        sign=packages_sign)
+                except BinhostSignError:
+                    for fn in (packages_fn, packages_sign_fn):
+                        if path.exists(fn):
+                            try:
+                                os.unlink(fn)
+                            except OSError:
+                                pass
+                    raise UpdateError(
+                        _("Current binhost {} has wrong signature").format(
+                            binhost_url))
+            else:
+                if binhost.bad_sign:
+                    raise UpdateError(
+                        _("Current binhost {} has wrong signature").format(
+                            binhost_url))
+        return True
+
+    def detect_best_binhost(self):
+        # выполняется переход с серверов unstable обновлней на stable
+        # в этом случае не важно, что бинари могут старее текущих
+        if (self.clVars.GetBool('cl_update_binhost_stable_opt_set') and
+                not self.clVars.GetBool('cl_update_binhost_stable_set')):
+            stabilization = True
+        else:
+            stabilization = False
 
+        self.startTask("Searching new binhost")
+        retval = self._search_best_binhost(self.binhosts_data, stabilization)
+
+        self.clVars.Set('update.cl_update_binhost_data',
+            retval or Variable.EmptyTable, force=True)
+
+        self.endTask()
+        return True
+
+    def check_sign_change(self):
+        if self.gpg_changed:
+            self.printWARNING(_("Public GPG key was updated"))
+            self.gpg_changed = False
+            self.clVars.Set('update.cl_update_outdate_set', Variable.On, force=True)
+            return True
+        return False

pym/update/utils/cl_update.py

@@ -15,7 +15,7 @@
 #  limitations under the License.
 
 import sys
-from calculate.core.server.func import Action, Tasks
+from calculate.core.server.func import Action, Tasks, AllTasks
 from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
 from calculate.lib.cl_template import TemplatesError
 from calculate.lib.utils.binhosts import BinhostError
@@ -195,33 +195,119 @@ class ClUpdateAction(Action):
         {'name': 'reps_synchronization',
          'group': __("Repositories synchronization"),
          'tasks': [
-             # запасная синхронизация, в ходе которой ветки обновляются до
+             # создать объект проверки PGP
-             # master
+             {'name': 'prepare_gpg',
-             {'name': 'sync_reps_fallback',
+              'method': 'Update.prepare_gpg()',
-              'foreach': 'cl_update_sync_rep',
+             },
-              'message':
+             # создать объект хранилище серверов обновлений
-                  __("Fallback syncing the {eachvar:capitalize} repository"),
+             {'name': 'create_binhost_data',
-              'method': 'Update.syncRepositories(eachvar,True)',
+              'method': 'Update.create_binhost_data()'
-              'condition': lambda Get,GetBool: (
+             },
-                    GetBool('cl_update_usetag_set') and
+             # проверить валиден ли текущий хост
-                    not Get('cl_update_binhost_data')[0])
+             {'name': 'check_current_binhost',
+              'message': __("Checking current binhost"),
+              'essential': False,
+              'method': 'Update.check_current_binhost(cl_update_binhost)',
+              'condition': lambda GetBool, Get: (
+                    not GetBool('cl_update_binhost_recheck_set') and
+                    Get('cl_update_sync_rep') and
+                    Get('cl_update_binhost'))
              },
-             # обновление переменных информации из binhost
+             {'name': 'group_find_binhost',
-             {'name': 'update_binhost_list',
+              'group': '',
-              'method': 'Update.update_binhost_list()',
+              'while': ((~AllTasks.has_any("detect_best_binhost") |
-              'condition': lambda Get,GetBool: (
+                         AllTasks.success_all("check_sign_change")) &
-                    GetBool('cl_update_usetag_set') and
+                         (AllTasks.failed_all("update_packages_cache") |
-                    not Get('cl_update_binhost_data')[0])
+                          ~AllTasks.has_any("sync_reps"))) & Tasks.success(),
+              'condition': lambda GetBool, Get: (GetBool('cl_update_usetag_set') and
+                                                 Get('cl_update_sync_rep')),
+              'tasks': [
+                 {'name': 'search_again_message',
+                  'warning': __("Binhost will be searched again"),
+                  'depend': AllTasks.success_all("check_sign_change")
+                 },
+                 # найти лучший сервер обновлений
+                 {'name': 'detect_best_binhost',
+                  'method': 'Update.detect_best_binhost()',
+                  'essential': False,
+                  'depend': Tasks.success() & (~AllTasks.success_one_of("check_current_binhost") |
+                             AllTasks.success_all("sync_reps")),
+                 },
+                 # запасная синхронизация, в ходе которой ветки обновляются до
+                 # master
+                 {'name': 'sync_reps_fallback',
+                  'foreach': 'cl_update_sync_rep',
+                  'message':
+                      __("Fallback syncing the {eachvar:capitalize} repository"),
+                  'method': 'Update.syncRepositories(eachvar,True)',
+                  'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"),
+                 },
+                 # обновление переменных информации из binhost
+                 {'name': 'sync_reps_fallback:update_binhost_list',
+                  'method': 'Update.update_binhost_list()',
+                  'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"),
+                 },
+                 # найти лучший сервер обновлений 
+                 {'name': 'sync_reps_fallback:detect_best_binhost',
+                  'method': 'Update.detect_best_binhost()',
+                  'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"),
+                 },
+                 {'name': 'sync_reps',
+                  'foreach': 'cl_update_sync_rep',
+                  'message': __("Checking {eachvar:capitalize} updates"),
+                  'method': 'Update.syncRepositories(eachvar)',
+                  'condition': lambda Get: Get('cl_update_sync_rep'),
+                  'depend': Tasks.success() & ~AllTasks.success_all("update_packages_cache")
+                 },
+                 {'name': 'sync_reps:update_local_info_binhost',
+                  'method': 'Update.update_local_info_binhost()',
+                 },
+                 {'name': 'sync_reps:update_binhost_list',
+                  'essential': False,
+                  'method': 'Update.update_binhost_list()',
+                  'condition': lambda GetBool: GetBool('cl_update_outdate_set')
+                 },
+                 # проверить изменились ли GPG ключи
+                 {'name': 'check_sign_change',
+                  'essential': False,
+                  'method': 'Update.check_sign_change()'
+                 },
+                 {'name': 'sync_reps:update_packages_cache',
+                  'message': __("Update packages index"),
+                  'method': 'Update.download_packages(cl_update_portage_binhost,'
+                            'cl_update_package_cache,cl_update_package_cache_sign,'
+                            'cl_update_gpg)',
+                  'essential': False,
+                  'condition': lambda Get, GetBool: (
+                      Get('cl_update_package_cache') and (
+                          Get('cl_update_outdate_set') == 'on' or
+                          Get('cl_update_package_cache_set') == 'on'))
+                 },
+                ],
+             },
+             {'name': 'no_server',
+              'error': __("Failed to find the server with appropriate updates"),
+              'method': "Update.delete_binhost()",
+              # method: который должен удалить текущую информацию о сервере обновлений
+              'depend': (Tasks.failed() | 
+                         Tasks.success() & AllTasks.failed_one_of("update_packages_cache")),
+              'condition': lambda GetBool, Get: (GetBool('cl_update_usetag_set') and
+                                                 Get('cl_update_sync_rep')),
              },
              {'name': 'sync_reps',
               'foreach': 'cl_update_sync_rep',
               'message': __("Checking {eachvar:capitalize} updates"),
               'method': 'Update.syncRepositories(eachvar)',
-              'condition': lambda Get: Get('cl_update_sync_rep')
+              'condition': lambda Get, GetBool: (Get('cl_update_sync_rep') and
+                                                 not GetBool('cl_update_usetag_set')),
+             },
+             # TODO:DEBUG
+             {'name': 'debug',
+              'error': "DEBUG:FAILED",
+              'depend': Tasks.failed()
              },
-             {'name': 'check_binhost',
+             {'name': 'debug',
-              'method': 'Update.check_binhost()',
+              'error': "DEBUG:ALL OK",
-              'condition': lambda GetBool: GetBool('cl_update_usetag_set')
              },
              {'name': 'sync_other_reps',
               'foreach': 'cl_update_other_rep_name',
@@ -282,17 +368,6 @@ class ClUpdateAction(Action):
                               Get('cl_update_outdate_set') == 'on'),
               'essential': False
              },
-             {'name': 'update_packages_cache',
-              'message': __("Update packages index"),
-              'method': 'Update.download_packages(cl_update_portage_binhost,'
-                        'cl_update_package_cache)',
-              'essential': False,
-              'condition': lambda Get, GetBool: (
-                  GetBool('cl_update_usetag_set') and
-                  Get('cl_update_package_cache') and (
-                      Get('cl_update_outdate_set') == 'on' or
-                      Get('cl_update_package_cache_set') == 'on'))
-              },
              # сообщение удачного завершения при обновлении репозиториев
              {'name': 'success_syncrep',
               'message': __("Synchronization finished"),

pym/update/variables/update.py

@@ -25,7 +25,8 @@ from calculate.lib.datavars import (Variable, VariableError,
                                     TableVariable, FieldValue,
                                     HumanReadable, CriticalError,
                                     SimpleDataVars, DataVarsError)
-from calculate.lib.utils.binhosts import Binhosts, PackagesIndex, HOURS
+from calculate.lib.utils.binhosts import (Binhosts, PackagesIndex, HOURS,
+    BinhostSignError)
 from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin
 
 from calculate.lib.configparser import ConfigParser
@@ -41,6 +42,7 @@ from calculate.lib.variables import env
 from calculate.update.update_info import UpdateInfo
 from itertools import chain
 from urlparse import urlparse
+import io
 import time
 
 _ = lambda x: x
@@ -277,108 +279,46 @@ class VariableClUpdateBranchRep(ReadonlyVariable):
     def get(self):
         return self.Get('cl_update_rep_name')
 
-
+class VariableClUpdateBinhostsInfo(ReadonlyVariable):
-class VariableClUpdateBinhostData(ReadonlyTableVariable):
     """
-    Таблица содержащая
+    Объект для получения информации о серверах обновлений
-    binhost/содержимое файла ревизий/время доступа
     """
-    source = ["cl_update_binhost_host",
+    type = Variable.Types.Object
-              "cl_update_binhost_revisions",
+    
-              "cl_update_binhost_timestamp",
+    def get(self):
-              "cl_update_binhost_time"]
-
-    def get_logger(self, stub=False):
-        class StubLogger(object):
-            def info(self, message):
-                pass
-
-        if not stub and (not self.Get('cl_ebuild_phase')):
-            # проверка на chroot_path
-            return log("binhost-scan.log",
-                       filename=pathJoin(
-                           self.Get('cl_chroot_path'),
-                           "/var/log/calculate/binhost-scan.log"),
-                       formatter="%(message)s")
-        else:
-            return StubLogger()
-
-    def get(self, hr=HumanReadable.No):
         last_ts = self.Get('cl_update_last_timestamp')
         if self.GetBool('cl_update_binhost_stable_opt_set'):
             binhost_list = self.Get('cl_update_binhost_list')
         else:
             binhost_list = self.Get('cl_update_binhost_unstable_list')
-        if (self.GetBool('cl_update_binhost_stable_opt_set') and
+        return Binhosts(
-                not self.GetBool('cl_update_binhost_stable_set')):
-            stabilization = True
-        else:
-            stabilization = False
-        binhosts_data = Binhosts(
             self.GetInteger('cl_update_binhost_timeout'),
             self.Get('cl_update_binhost_revision_path'),
             self.Get('cl_update_binhost_timestamp_path'),
-            last_ts,
+            last_ts, binhost_list,
-            binhost_list
+            self.Get('os_arch_machine'),
-        )
+            gpg=self.Get('cl_update_gpg'))
-        binhost = self.Get('cl_update_binhost')
-        recheck = self.GetBool('cl_update_binhost_recheck_set')
-
-        if not recheck and binhost and binhost in binhost_list:
-            ts, t, good, downgrade = binhosts_data.get_timestamp(binhost)
-            # условие актуальности текущего сервера
-            if ts and good and not downgrade:
-                status, data = binhosts_data.check_binhost(binhost)
-                if status == binhosts_data.BinhostStatus.Success:
-                    return [[binhost, data, ts, str(t)]]
-
-        logger = self.get_logger(stub=binhosts_data.is_cache())
-        ret_val = None
-        logger.info("Started scan on: {date}, current timestamp: {ts}".format(
-            date=time.ctime(), ts=last_ts))
-        for host, ts, t, good, downgrade in binhosts_data.get_sorted():
-            if ret_val is None:
-                status, data = binhosts_data.check_binhost(host)
-            else:
-                status, data = binhosts_data.BinhostStatus.UnknownError, ""
 
-            if not good:
+class VariableClUpdateBinhostData(ReadonlyTableVariable):
-                if ts == "0":
+    """
-                    reason = "FAILED (Wrong binhost)"
+    Таблица содержащая
-                    data = ""
+    binhost/содержимое файла ревизий/время доступа
-                    t = 0
+    """
-                else:
+    source = ["cl_update_binhost_host",
-                    reason = "OUTDATED"
+              "cl_update_binhost_revisions",
-            elif downgrade:
+              "cl_update_binhost_timestamp",
-                reason = "SKIP"
+              "cl_update_binhost_time"]
-            elif not data and ret_val is None:
-                errors = {
-                    binhosts_data.BinhostStatus.Updating: "UPDATING",
-                    binhosts_data.BinhostStatus.BadEnv:
-                        "FAILED (Bad env)",
-                    binhosts_data.BinhostStatus.EnvNotFound:
-                        "FAILED (Env not found)",
-                }
-                reason = errors.get(status, "FAILED (Unknown)")
-            else:
-                reason = ""
 
-            if ret_val is None and data and (
+    def get(self, hr=HumanReadable.No):
-                        not downgrade or stabilization):
+        return Variable.EmptyTable
-                ret_val = [[host, data, ts, str(t)]]
-                host = "-> %s" % host
-            logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format(
-                host=host, speed=float(t) / 1000.0, timestamp=ts,
-                reason=reason))
-        return ret_val or [[]]
 
 
 class VariableClUpdateBinhostRecheckSet(Variable):
     """
     Принудительно обновить binhost
     """
-    type = "bool"
+    type = Variable.Types.Boolean
-    value = "off"
+    value = Variable.Off
 
     opt = ["--scan"]
 
@@ -387,7 +327,7 @@ class VariableClUpdateBinhostRecheckSet(Variable):
         self.label = _("Search for the most appropriate update server")
 
 
-class VariableClUpdateBinhostHost(FieldValue, ReadonlyVariable):
+class VariableClUpdateBinhostHost(FieldValue, Variable):
     """
     Список имен прочих репозиториев
     """
@@ -396,7 +336,7 @@ class VariableClUpdateBinhostHost(FieldValue, ReadonlyVariable):
     column = 0
 
 
-class VariableClUpdateBinhostRevisions(FieldValue, ReadonlyVariable):
+class VariableClUpdateBinhostRevisions(FieldValue, Variable):
     """
     Список имен прочих репозиториев
     """
@@ -405,7 +345,7 @@ class VariableClUpdateBinhostRevisions(FieldValue, ReadonlyVariable):
     column = 1
 
 
-class VariableClUpdateBinhostTimestamp(FieldValue, ReadonlyVariable):
+class VariableClUpdateBinhostTimestamp(FieldValue, Variable):
     """
     Список имен прочих репозиториев
     """
@@ -414,6 +354,15 @@ class VariableClUpdateBinhostTimestamp(FieldValue, ReadonlyVariable):
     column = 2
 
 
+class VariableClUpdateBinhostTime(FieldValue, Variable):
+    """
+    Список имен прочих репозиториев
+    """
+    type = "list"
+    source_variable = "cl_update_binhost_data"
+    column = 3
+
+
 class VariableClUpdateLastTimestamp(ReadonlyVariable):
     """
     Текущий timestamp
@@ -424,15 +373,6 @@ class VariableClUpdateLastTimestamp(ReadonlyVariable):
         return ini.getVar('system', 'last_update') or "0"
 
 
-class VariableClUpdateBinhostTime(FieldValue, ReadonlyVariable):
-    """
-    Список имен прочих репозиториев
-    """
-    type = "list"
-    source_variable = "cl_update_binhost_data"
-    column = 3
-
-
 class VariableClUpdateBranchName(Variable):
     """
     Список доступных репозиторием
@@ -586,6 +526,29 @@ class VariableClUpdateEixupdateForce(Variable):
                 ("auto", _("If needed"))]
 
 
+class VariableClUpdateGpgForce(Variable):
+    """
+    Принудительное действие с eix-update
+    """
+    type = "choice"
+    value = "auto"
+    opt = ["--check-sign"]
+    syntax = "--{choice}-check-sign"
+    metavalue = "MODE"
+    # untrusted = True
+
+    def init(self):
+        self.help = ("'force' - " + _("force check Package signature") +
+                     ",\n'skip' - " + _("skip check Package signature") +
+                     ",\n'auto' - " + _("check signature if system has public keys"))
+        self.label = _("Check Package signature")
+
+    def choice(self):
+        return [("force", _("Force")),
+                ("skip", _("Skip")),
+                ("auto", _("If has passibility"))]
+
+
 class VariableClUpdateOtherSet(Variable):
     """
     Обновить остальные оверлеи
@@ -811,9 +774,6 @@ class VariableClUpdateProfileDependData(ReadonlyTableVariable):
 
     def get(self, hr=HumanReadable.No):
         dv = self.Get(self.datavars)
-        # TODO:  неиспользуемая переменная возможно
-        # испольуется для инициализации
-        # url = self.Get('cl_update_profile_url').lower()
         if dv:
             if hr == HumanReadable.Yes:
                 return reversed(zip([x.capitalize() for x in dv.Get('cl_update_rep_name')],
@@ -1076,17 +1036,6 @@ class VariableClUpdateProfileBranch(Variable):
         self.label = _("Repository branch")
         self.help = _("set the repository branch")
 
-    def check(self, value):
-        pass
-        ## TODO: проверка ветки
-        # try:
-        #    url = self.Get('cl_update_profile_url')
-        #    self.Get('cl_update_profile_storage').get_profiles(url, value)
-        # except GitError as e:
-
-
-# raise VariableError(str(e))
-
 
 class VariableClProfileRepository(ReadonlyVariable):
     """
@@ -1736,6 +1685,12 @@ class VariableClUpdatePackageCache(ReadonlyVariable):
                             host, parsed_url.path.lstrip("/"),
                             "Packages")
 
+class VariableClUpdatePackageCacheSign(ReadonlyVariable):
+    """
+    Файл с подписью Packages находящийся в кэше
+    """
+    value_format = "{update.cl_update_package_cache}.asc"
+
 
 class VariableClUpdatePackageCacheSet(Variable):
     """
@@ -1745,7 +1700,9 @@ class VariableClUpdatePackageCacheSet(Variable):
 
     def get(self):
         packages_fn = self.Get('cl_update_package_cache')
-        if not path.exists(packages_fn):
+        packages_asc_fn = self.Get('cl_update_package_cache_sign')
+        if (not path.exists(packages_fn) or
+            not path.exists(packages_asc_fn)):
             return "on"
         pi = PackagesIndex(readFile(packages_fn))
         try:
@@ -1800,3 +1757,42 @@ class VariableClUpdateSshkeyPath(Variable):
     Путь до ssh ключа
     """
     value = "/var/lib/calculate/id_rsa"
+
+class VariableClUpdateBadSignSet(Variable):
+    """
+    Ошибка проверки подписи Packages
+    """
+    type = Variable.Types.Boolean
+    value = Variable.Off
+
+class VariableClUpdateGpgKeys(Variable):
+    """
+    Список ключей для проверки подписи Packages
+    """
+    type = Variable.Types.List
+    value = ["/usr/share/openpgp-keys/calculate-release.asc"]
+
+class VariableClUpdateGpg(ReadonlyVariable):
+    """
+    Объект проверки подписи
+    """
+    type = Variable.Types.Object
+    value = ""
+
+    def close(self):
+        val = self.Get()
+        if val:
+            val.close()
+
+class VariableClUpdateUseDowngradeSet(ReadonlyVariable):
+    """
+    Можно использовать downgrade binhost
+    """
+    type = Variable.Types.Boolean
+
+    def get(self):
+        """
+        Выполняется переход с unstable ветки на stable
+        """
+        return (self.GetBool("cl_update_binhost_stable_opt_set") and 
+                not self.GetBool("cl_update_binhost_stable_opt_set"))

pym/update/wsdl_update.py

@@ -78,6 +78,7 @@ class Wsdl(WsdlBase):
                                     'cl_update_check_rep_set',
                                     'cl_update_force_fix_set',
                                     'cl_update_world',
+                                    'cl_update_gpg_force',
                                     'cl_update_egencache_force',
                                     'cl_update_eixupdate_force',
                                     'cl_update_skip_rb_set',