diff --git a/pym/update/update.py b/pym/update/update.py index e5f8643..ab4b8e5 100644 --- a/pym/update/update.py +++ b/pym/update/update.py @@ -24,7 +24,7 @@ from calculate.core.server.gen_pid import search_worked_process from calculate.core.setup_cache import Cache as SetupCache from calculate.core.server.func import MethodsInterface from calculate.lib.cl_template import SystemIni, LayeredIni -from calculate.lib.datavars import DataVarsError, VariableError +from calculate.lib.datavars import DataVarsError, VariableError, Variable from calculate.lib.utils.tools import AddonError from calculate.lib.utils.colortext.palette import TextState @@ -32,12 +32,16 @@ from calculate.lib.utils.colortext import get_color_print from calculate.update.emerge_parser import RevdepPercentBlock from calculate.update.datavars import DataVarsUpdate from calculate.update.update_info import UpdateInfo -from calculate.lib.utils.binhosts import fetch_packages +from calculate.lib.utils.binhosts import (Binhosts, BinhostSignError, + BinhostError, PackagesIndex, DAYS) +from calculate.lib.utils.gpg import GPG, GPGError from calculate.lib.cl_log import log +import hashlib import re import shutil from collections import MutableSet from contextlib import contextmanager +import tempfile from calculate.lib.utils.git import Git, GitError, MTimeKeeper, NotGitError from calculate.lib.utils.portage import (Layman, EmergeLog, @@ -52,7 +56,7 @@ Colors = TextState.Colors from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir, PercentProgress, process, getRunCommands, readFile, listDirectory, pathJoin, - find, FindFileType, + find, FindFileType,quite_unlink, writeFile, makeDirectory) import emerge_parser import logging @@ -151,6 +155,8 @@ class Update(MethodsInterface): self.refresh_binhost = False self.pkgnum = None self.pkgnummax = None + self.gpgdata_md5 = [] + self.gpg_changed = False def get_prog_path(self, program_name): return getProgPath(program_name) @@ -217,7 +223,7 @@ class Update(MethodsInterface): status = git.getStatusInfo(rpath) if not status or status['files']: need_update = True - except GitError: + except GitError as e: need_update = True if need_update: if not notask: @@ -236,6 +242,8 @@ class Update(MethodsInterface): dv.Set('cl_update_outdate_set', 'on', force=True) finally: self.unstash_cache(rpath, name) + # TODO: debug1 + #dv.Set('cl_update_outdate_set', 'on', force=True) return True def raiseOutdate(self): @@ -1213,28 +1221,23 @@ class Update(MethodsInterface): cache.update(force=True) return True - def check_binhost(self, write_binhost=True): + def get_bin_cache_filename(self): + return pathJoin(self.clVars.Get('cl_chroot_path'), + LayeredIni.IniPath.Grp) + + def update_local_info_binhost(self, write_binhost=True): """ Проверить, что доступен хотя бы один из binhost'ов :return: """ hosts = self.clVars.Get("update.cl_update_binhost_host") datas = self.clVars.Get("update.cl_update_binhost_revisions") - bin_cache_fn = pathJoin(self.clVars.Get('cl_chroot_path'), - LayeredIni.IniPath.Grp) if not hosts: - self.clVars.Delete('cl_update_binhost', location="system") - try: - if path.exists(bin_cache_fn): - os.unlink(bin_cache_fn) - except OSError: - raise UpdateError( - _("Failed to remove cached ini.env of binary repository")) - + self.delete_binhost() raise UpdateError(_("Failed to find the server with " "appropriate updates")) else: - with writeFile(bin_cache_fn) as f: + with writeFile(self.get_bin_cache_filename()) as f: f.write(datas[0].strip()+"\n") if write_binhost: if hosts[0] != self.clVars.Get('update.cl_update_binhost'): @@ -1275,33 +1278,53 @@ class Update(MethodsInterface): self.clVars.Get('cl_update_binhost')) return True + def delete_binhost(self): + self.clVars.Delete('cl_update_binhost', location="system") + try: + bin_cache_fn = self.get_bin_cache_filename() + if path.exists(bin_cache_fn): + os.unlink(bin_cache_fn) + except OSError: + raise UpdateError( + _("Failed to remove cached ini.env of binary repository")) + try: + for varname in ('cl_update_package_cache', 'cl_update_package_cache_sign'): + fn = self.clVars.Get(varname) + if path.exists(fn): + os.unlink(fn) + except OSError: + raise UpdateError( + _("Failed to remove cached Package index")) + return True + def update_binhost_list(self): """ Обновить список binhost'ов после обновления до master веток :return: """ - dv = DataVarsUpdate() + changes = False try: + dv = DataVarsUpdate() dv.importUpdate() dv.flIniFile() - changes = False for varname in ('update.cl_update_binhost_list', 'update.cl_update_binhost_unstable_list', 'update.cl_update_binhost_timestamp_path', + 'update.cl_update_gpg_keys', 'cl_update_binhost_revision_path'): new_value = dv.Get(varname) old_value = self.clVars.Get(varname) if new_value != old_value: changes = True - self.clVars.Set(varname, new_value, force=True) - if not changes: - self.clVars.Delete('cl_update_binhost', location="system") - raise UpdateError(_("Failed to find the server with " - "appropriate updates")) + self.clVars.Set(varname, new_value, force=True) except DataVarsError: - self.clVars.Delete('cl_update_binhost', location="system") - raise UpdateError(_("Failed to find the server with " - "appropriate updates")) + raise UpdateError(_("Failed to get values for binhost search")) + + if self.is_gpg_changed(): + self.prepare_gpg() + elif not changes: + return False + self.create_binhost_data() return True def drop_binhosts(self, dv): @@ -1316,8 +1339,271 @@ class Update(MethodsInterface): dv.Invalidate('update.cl_update_rep_rev') return True - def download_packages(self, url_binhost, packages_fn): - fetch_packages(url_binhost, packages_fn) + def is_gpg_changed(self): + """ + Проверить изменились ли открытые ключи + """ + gpg_force = self.clVars.Get('cl_update_gpg_force') + if gpg_force == "skip": + return False + gpg_keys = self.clVars.Get('cl_update_gpg_keys') + gpgdata = "" + for keyfn in gpg_keys: + if path.exists(keyfn): + gpgdata += readFile(keyfn) + new_gpgdata_md5 = hashlib.md5(gpgdata).hexdigest() + return all(new_gpgdata_md5 != x for x in self.gpgdata_md5) + + def prepare_gpg(self): + """ + Получить объект для проверки подписи, либо получить заглушку + """ + gpg_force = self.clVars.Get('cl_update_gpg_force') + gpg_keys = self.clVars.Get('cl_update_gpg_keys') + if gpg_force == "skip": + return True + gpg = GPG(tempfile.mkdtemp(dir="/var/calculate/tmp/update", + prefix="gpg-")) + gpgdata = "" + for keyfn in gpg_keys: + if path.exists(keyfn): + try: + key = readFile(keyfn) + gpgdata += key + gpg.import_key(key) + except GPGError as e: + self.printWARNING(_("Failed to load public keys from '%s' " + "for signature checking") % keyfn) + self.gpgdata_md5.append(hashlib.md5(gpgdata).hexdigest()) + if len(self.gpgdata_md5) > 1: + self.gpg_changed = True + if not gpg.count_public(): + if gpg_force == "force": + raise UpdateError(_("Public keys for Packages signature checking not found")) + else: + return True + oldgpg = self.clVars.Get('update.cl_update_gpg') + if oldgpg: + oldgpg.close() + self.clVars.Set('update.cl_update_gpg', gpg, force=True) + return True + + def download_packages(self, url_binhost, packages_fn, packages_sign_fn, gpg): + quite_unlink(packages_fn) + orig_packages = Binhosts.fetch_packages(url_binhost) + try: + with writeFile(packages_fn) as f: + pi = PackagesIndex(orig_packages) + pi["TTL"] = str(30 * DAYS) + pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time())) + pi.write(f) + with writeFile("/tmp/Packages.org") as f: + f.write(orig_packages) + #pi = PackagesIndex(orig_packages) + #pi.write(f) + except (OSError, IOError): + raise UpdateError(_("Failed to save Packages")) + self.endTask(True) + self.startTask(_("Check packages index signature")) + if not gpg: + self.endTask("skip") + self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True) + return True + try: + Binhosts.check_packages_signature( + url_binhost, orig_packages, gpg) + with writeFile(packages_sign_fn) as f: + f.write(Binhosts.fetch_packages_sign(url_binhost)) + except BinhostSignError: + for fn in (packages_fn, packages_sign_fn): + if path.exists(fn): + try: + os.unlink(fn) + except OSError: + pass + self.clVars.Set("update.cl_update_bad_sign_set", Variable.On) + self.clVars.Set('update.cl_update_binhost_recheck_set', Variable.On) + self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True) + raise + return True + + class Reason(object): + WrongBinhost = "wrong_binhost" + Outdated = "outdated" + Updating = "updating" + BadEnv = "badenv" + EnvNotFound = "noenv" + UnknownError = "unknown" + BadSign = "badsign" + Skip = "skip" + SkipSlower = "skipslower" + Success = "success" + + @staticmethod + def humanReadable(reason): + return { + Update.Reason.WrongBinhost: "FAILED (Wrong binhost)", + Update.Reason.Outdated: "OUTDATED", + Update.Reason.Updating: "UPDATING", + Update.Reason.BadEnv: "FAILED (Bad env)", + Update.Reason.EnvNotFound: "FAILED (Env not found)", + Update.Reason.UnknownError: "FAILED (Unknown error)", + Update.Reason.BadSign: "FAILED (Bad sign)", + Update.Reason.Skip: "SKIP", + Update.Reason.SkipSlower: "", + Update.Reason.Success: "" + }.get(reason,reason) + + def _get_binhost_logger(self): + return log("binhost-scan.log", + filename=pathJoin( + self.clVars.Get('cl_chroot_path'), + "/var/log/calculate/binhost-scan.log"), + formatter="%(message)s") + + def create_binhost_data(self): + dv = self.clVars + last_ts = dv.Get('cl_update_last_timestamp') + if dv.GetBool('cl_update_binhost_stable_opt_set'): + binhost_list = dv.Get('cl_update_binhost_list') + else: + binhost_list = dv.Get('cl_update_binhost_unstable_list') + self.binhosts_data = Binhosts( + dv.GetInteger('cl_update_binhost_timeout'), + dv.Get('cl_update_binhost_revision_path'), + dv.Get('cl_update_binhost_timestamp_path'), + last_ts, binhost_list, + dv.Get('os_arch_machine'), + gpg=dv.Get('cl_update_gpg')) return True + def _search_best_binhost(self, binhosts_data, stabilization): + if not self.clVars.Get('cl_ebuild_phase'): + logger = self._get_binhost_logger() + if logger: + logger.info( + "Started scan on: {date}, current timestamp: {ts}".format( + date=time.ctime(), ts=binhosts_data.last_ts)) + retval = [] + skip_check_status = False + for binhost in sorted(binhosts_data.get_binhosts(), reverse=True): + host = binhost.host + if not binhost.valid: + reason = self.Reason.WrongBinhost + elif binhost.outdated: + reason = self.Reason.Outdated + elif not skip_check_status: + status = binhost.status + if status is not binhosts_data.BinhostStatus.Success: + errors = { + binhosts_data.BinhostStatus.Updating: self.Reason.Updating, + binhosts_data.BinhostStatus.BadEnv: self.Reason.BadEnv, + binhosts_data.BinhostStatus.EnvNotFound: self.Reason.EnvNotFound + } + reason = errors.get(status, self.Reason.UnknownError) + elif binhost.bad_sign: + reason = self.Reason.BadSign + else: + # SUCCESS + if not binhost.downgraded or stabilization: + host = "-> %s" % host + reason = self.Reason.Success + else: + reason = self.Reason.Skip + elif binhost.downgraded: + reason = self.Reason.Skip + else: + reason = self.Reason.SkipSlower + + if reason == self.Reason.Success: + retval.append([binhost.host, binhost.data, + str(binhost.timestamp), + str(binhost.duration)]) + skip_check_status = True + logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format( + host=host, speed=float(binhost.duration) / 1000.0, + timestamp=binhost.timestamp, + reason=Update.Reason.humanReadable(reason))) + if not retval: + raise UpdateError(_("Failed to find the server with appropriate updates")) + return retval + + def check_current_binhost(self, binhost_url): + """ + Проверка текущего сервера обновлений на валидность + """ + if not binhost_url in self.binhosts_data.binhost_list: + raise UpdateError(_("Current binhost is absent in list of update servers")) + binhost = self.binhosts_data.get_binhost(binhost_url) + + if binhost.valid and not binhost.outdated and not binhost.downgraded: + if binhost.status == self.binhosts_data.BinhostStatus.Success: + self.clVars.Set('update.cl_update_binhost_data', + [[binhost.host, binhost.data, + str(binhost.timestamp), + str(binhost.duration)]], + force=True) + self.endTask() + else: + if not binhost.valid: + raise UpdateError( + _("Current binhost {} is not valid").format(binhost_url)) + elif binhost.outdated: + raise UpdateError( + _("Current binhost {} is outdated").format(binhost_url)) + elif binhost.downgraded: + raise UpdateError( + _("Current binhost {} was downgraded").format(binhost_url)) + if self.binhosts_data.gpg: + packages_fn = self.clVars.Get('cl_update_package_cache') + packages_sign_fn = self.clVars.Get('cl_update_package_cache_sign') + if path.exists(packages_fn) and path.exists(packages_sign_fn): + packages_sign = readFile(packages_sign_fn) + pi = PackagesIndex(readFile(packages_fn)) + pi.clean() + try: + Binhosts.check_packages_signature( + None, pi.get_value(), self.binhosts_data.gpg, + sign=packages_sign) + except BinhostSignError: + for fn in (packages_fn, packages_sign_fn): + if path.exists(fn): + try: + os.unlink(fn) + except OSError: + pass + raise UpdateError( + _("Current binhost {} has wrong signature").format( + binhost_url)) + else: + if binhost.bad_sign: + raise UpdateError( + _("Current binhost {} has wrong signature").format( + binhost_url)) + return True + + def detect_best_binhost(self): + # выполняется переход с серверов unstable обновлней на stable + # в этом случае не важно, что бинари могут старее текущих + if (self.clVars.GetBool('cl_update_binhost_stable_opt_set') and + not self.clVars.GetBool('cl_update_binhost_stable_set')): + stabilization = True + else: + stabilization = False + self.startTask("Searching new binhost") + retval = self._search_best_binhost(self.binhosts_data, stabilization) + + self.clVars.Set('update.cl_update_binhost_data', + retval or Variable.EmptyTable, force=True) + + self.endTask() + return True + + def check_sign_change(self): + if self.gpg_changed: + self.printWARNING(_("Public GPG key was updated")) + self.gpg_changed = False + self.clVars.Set('update.cl_update_outdate_set', Variable.On, force=True) + return True + return False diff --git a/pym/update/utils/cl_update.py b/pym/update/utils/cl_update.py index e98ca6c..5af18d6 100644 --- a/pym/update/utils/cl_update.py +++ b/pym/update/utils/cl_update.py @@ -15,7 +15,7 @@ # limitations under the License. import sys -from calculate.core.server.func import Action, Tasks +from calculate.core.server.func import Action, Tasks, AllTasks from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate from calculate.lib.cl_template import TemplatesError from calculate.lib.utils.binhosts import BinhostError @@ -195,33 +195,119 @@ class ClUpdateAction(Action): {'name': 'reps_synchronization', 'group': __("Repositories synchronization"), 'tasks': [ - # запасная синхронизация, в ходе которой ветки обновляются до - # master - {'name': 'sync_reps_fallback', - 'foreach': 'cl_update_sync_rep', - 'message': - __("Fallback syncing the {eachvar:capitalize} repository"), - 'method': 'Update.syncRepositories(eachvar,True)', - 'condition': lambda Get,GetBool: ( - GetBool('cl_update_usetag_set') and - not Get('cl_update_binhost_data')[0]) + # создать объект проверки PGP + {'name': 'prepare_gpg', + 'method': 'Update.prepare_gpg()', + }, + # создать объект хранилище серверов обновлений + {'name': 'create_binhost_data', + 'method': 'Update.create_binhost_data()' + }, + # проверить валиден ли текущий хост + {'name': 'check_current_binhost', + 'message': __("Checking current binhost"), + 'essential': False, + 'method': 'Update.check_current_binhost(cl_update_binhost)', + 'condition': lambda GetBool, Get: ( + not GetBool('cl_update_binhost_recheck_set') and + Get('cl_update_sync_rep') and + Get('cl_update_binhost')) }, - # обновление переменных информации из binhost - {'name': 'update_binhost_list', - 'method': 'Update.update_binhost_list()', - 'condition': lambda Get,GetBool: ( - GetBool('cl_update_usetag_set') and - not Get('cl_update_binhost_data')[0]) + {'name': 'group_find_binhost', + 'group': '', + 'while': ((~AllTasks.has_any("detect_best_binhost") | + AllTasks.success_all("check_sign_change")) & + (AllTasks.failed_all("update_packages_cache") | + ~AllTasks.has_any("sync_reps"))) & Tasks.success(), + 'condition': lambda GetBool, Get: (GetBool('cl_update_usetag_set') and + Get('cl_update_sync_rep')), + 'tasks': [ + {'name': 'search_again_message', + 'warning': __("Binhost will be searched again"), + 'depend': AllTasks.success_all("check_sign_change") + }, + # найти лучший сервер обновлений + {'name': 'detect_best_binhost', + 'method': 'Update.detect_best_binhost()', + 'essential': False, + 'depend': Tasks.success() & (~AllTasks.success_one_of("check_current_binhost") | + AllTasks.success_all("sync_reps")), + }, + # запасная синхронизация, в ходе которой ветки обновляются до + # master + {'name': 'sync_reps_fallback', + 'foreach': 'cl_update_sync_rep', + 'message': + __("Fallback syncing the {eachvar:capitalize} repository"), + 'method': 'Update.syncRepositories(eachvar,True)', + 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), + }, + # обновление переменных информации из binhost + {'name': 'sync_reps_fallback:update_binhost_list', + 'method': 'Update.update_binhost_list()', + 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), + }, + # найти лучший сервер обновлений + {'name': 'sync_reps_fallback:detect_best_binhost', + 'method': 'Update.detect_best_binhost()', + 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), + }, + {'name': 'sync_reps', + 'foreach': 'cl_update_sync_rep', + 'message': __("Checking {eachvar:capitalize} updates"), + 'method': 'Update.syncRepositories(eachvar)', + 'condition': lambda Get: Get('cl_update_sync_rep'), + 'depend': Tasks.success() & ~AllTasks.success_all("update_packages_cache") + }, + {'name': 'sync_reps:update_local_info_binhost', + 'method': 'Update.update_local_info_binhost()', + }, + {'name': 'sync_reps:update_binhost_list', + 'essential': False, + 'method': 'Update.update_binhost_list()', + 'condition': lambda GetBool: GetBool('cl_update_outdate_set') + }, + # проверить изменились ли GPG ключи + {'name': 'check_sign_change', + 'essential': False, + 'method': 'Update.check_sign_change()' + }, + {'name': 'sync_reps:update_packages_cache', + 'message': __("Update packages index"), + 'method': 'Update.download_packages(cl_update_portage_binhost,' + 'cl_update_package_cache,cl_update_package_cache_sign,' + 'cl_update_gpg)', + 'essential': False, + 'condition': lambda Get, GetBool: ( + Get('cl_update_package_cache') and ( + Get('cl_update_outdate_set') == 'on' or + Get('cl_update_package_cache_set') == 'on')) + }, + ], + }, + {'name': 'no_server', + 'error': __("Failed to find the server with appropriate updates"), + 'method': "Update.delete_binhost()", + # method: который должен удалить текущую информацию о сервере обновлений + 'depend': (Tasks.failed() | + Tasks.success() & AllTasks.failed_one_of("update_packages_cache")), + 'condition': lambda GetBool, Get: (GetBool('cl_update_usetag_set') and + Get('cl_update_sync_rep')), }, {'name': 'sync_reps', 'foreach': 'cl_update_sync_rep', 'message': __("Checking {eachvar:capitalize} updates"), 'method': 'Update.syncRepositories(eachvar)', - 'condition': lambda Get: Get('cl_update_sync_rep') + 'condition': lambda Get, GetBool: (Get('cl_update_sync_rep') and + not GetBool('cl_update_usetag_set')), + }, + # TODO:DEBUG + {'name': 'debug', + 'error': "DEBUG:FAILED", + 'depend': Tasks.failed() }, - {'name': 'check_binhost', - 'method': 'Update.check_binhost()', - 'condition': lambda GetBool: GetBool('cl_update_usetag_set') + {'name': 'debug', + 'error': "DEBUG:ALL OK", }, {'name': 'sync_other_reps', 'foreach': 'cl_update_other_rep_name', @@ -282,17 +368,6 @@ class ClUpdateAction(Action): Get('cl_update_outdate_set') == 'on'), 'essential': False }, - {'name': 'update_packages_cache', - 'message': __("Update packages index"), - 'method': 'Update.download_packages(cl_update_portage_binhost,' - 'cl_update_package_cache)', - 'essential': False, - 'condition': lambda Get, GetBool: ( - GetBool('cl_update_usetag_set') and - Get('cl_update_package_cache') and ( - Get('cl_update_outdate_set') == 'on' or - Get('cl_update_package_cache_set') == 'on')) - }, # сообщение удачного завершения при обновлении репозиториев {'name': 'success_syncrep', 'message': __("Synchronization finished"), diff --git a/pym/update/variables/update.py b/pym/update/variables/update.py index 1c8cc05..8e9b62e 100644 --- a/pym/update/variables/update.py +++ b/pym/update/variables/update.py @@ -25,7 +25,8 @@ from calculate.lib.datavars import (Variable, VariableError, TableVariable, FieldValue, HumanReadable, CriticalError, SimpleDataVars, DataVarsError) -from calculate.lib.utils.binhosts import Binhosts, PackagesIndex, HOURS +from calculate.lib.utils.binhosts import (Binhosts, PackagesIndex, HOURS, + BinhostSignError) from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin from calculate.lib.configparser import ConfigParser @@ -41,6 +42,7 @@ from calculate.lib.variables import env from calculate.update.update_info import UpdateInfo from itertools import chain from urlparse import urlparse +import io import time _ = lambda x: x @@ -277,108 +279,46 @@ class VariableClUpdateBranchRep(ReadonlyVariable): def get(self): return self.Get('cl_update_rep_name') - -class VariableClUpdateBinhostData(ReadonlyTableVariable): +class VariableClUpdateBinhostsInfo(ReadonlyVariable): """ - Таблица содержащая - binhost/содержимое файла ревизий/время доступа + Объект для получения информации о серверах обновлений """ - source = ["cl_update_binhost_host", - "cl_update_binhost_revisions", - "cl_update_binhost_timestamp", - "cl_update_binhost_time"] - - def get_logger(self, stub=False): - class StubLogger(object): - def info(self, message): - pass - - if not stub and (not self.Get('cl_ebuild_phase')): - # проверка на chroot_path - return log("binhost-scan.log", - filename=pathJoin( - self.Get('cl_chroot_path'), - "/var/log/calculate/binhost-scan.log"), - formatter="%(message)s") - else: - return StubLogger() - - def get(self, hr=HumanReadable.No): + type = Variable.Types.Object + + def get(self): last_ts = self.Get('cl_update_last_timestamp') if self.GetBool('cl_update_binhost_stable_opt_set'): binhost_list = self.Get('cl_update_binhost_list') else: binhost_list = self.Get('cl_update_binhost_unstable_list') - if (self.GetBool('cl_update_binhost_stable_opt_set') and - not self.GetBool('cl_update_binhost_stable_set')): - stabilization = True - else: - stabilization = False - binhosts_data = Binhosts( + return Binhosts( self.GetInteger('cl_update_binhost_timeout'), self.Get('cl_update_binhost_revision_path'), self.Get('cl_update_binhost_timestamp_path'), - last_ts, - binhost_list - ) - binhost = self.Get('cl_update_binhost') - recheck = self.GetBool('cl_update_binhost_recheck_set') - - if not recheck and binhost and binhost in binhost_list: - ts, t, good, downgrade = binhosts_data.get_timestamp(binhost) - # условие актуальности текущего сервера - if ts and good and not downgrade: - status, data = binhosts_data.check_binhost(binhost) - if status == binhosts_data.BinhostStatus.Success: - return [[binhost, data, ts, str(t)]] - - logger = self.get_logger(stub=binhosts_data.is_cache()) - ret_val = None - logger.info("Started scan on: {date}, current timestamp: {ts}".format( - date=time.ctime(), ts=last_ts)) - for host, ts, t, good, downgrade in binhosts_data.get_sorted(): - if ret_val is None: - status, data = binhosts_data.check_binhost(host) - else: - status, data = binhosts_data.BinhostStatus.UnknownError, "" + last_ts, binhost_list, + self.Get('os_arch_machine'), + gpg=self.Get('cl_update_gpg')) - if not good: - if ts == "0": - reason = "FAILED (Wrong binhost)" - data = "" - t = 0 - else: - reason = "OUTDATED" - elif downgrade: - reason = "SKIP" - elif not data and ret_val is None: - errors = { - binhosts_data.BinhostStatus.Updating: "UPDATING", - binhosts_data.BinhostStatus.BadEnv: - "FAILED (Bad env)", - binhosts_data.BinhostStatus.EnvNotFound: - "FAILED (Env not found)", - } - reason = errors.get(status, "FAILED (Unknown)") - else: - reason = "" +class VariableClUpdateBinhostData(ReadonlyTableVariable): + """ + Таблица содержащая + binhost/содержимое файла ревизий/время доступа + """ + source = ["cl_update_binhost_host", + "cl_update_binhost_revisions", + "cl_update_binhost_timestamp", + "cl_update_binhost_time"] - if ret_val is None and data and ( - not downgrade or stabilization): - ret_val = [[host, data, ts, str(t)]] - host = "-> %s" % host - logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format( - host=host, speed=float(t) / 1000.0, timestamp=ts, - reason=reason)) - return ret_val or [[]] + def get(self, hr=HumanReadable.No): + return Variable.EmptyTable class VariableClUpdateBinhostRecheckSet(Variable): """ Принудительно обновить binhost """ - type = "bool" - value = "off" + type = Variable.Types.Boolean + value = Variable.Off opt = ["--scan"] @@ -387,7 +327,7 @@ class VariableClUpdateBinhostRecheckSet(Variable): self.label = _("Search for the most appropriate update server") -class VariableClUpdateBinhostHost(FieldValue, ReadonlyVariable): +class VariableClUpdateBinhostHost(FieldValue, Variable): """ Список имен прочих репозиториев """ @@ -396,7 +336,7 @@ class VariableClUpdateBinhostHost(FieldValue, ReadonlyVariable): column = 0 -class VariableClUpdateBinhostRevisions(FieldValue, ReadonlyVariable): +class VariableClUpdateBinhostRevisions(FieldValue, Variable): """ Список имен прочих репозиториев """ @@ -405,7 +345,7 @@ class VariableClUpdateBinhostRevisions(FieldValue, ReadonlyVariable): column = 1 -class VariableClUpdateBinhostTimestamp(FieldValue, ReadonlyVariable): +class VariableClUpdateBinhostTimestamp(FieldValue, Variable): """ Список имен прочих репозиториев """ @@ -414,6 +354,15 @@ class VariableClUpdateBinhostTimestamp(FieldValue, ReadonlyVariable): column = 2 +class VariableClUpdateBinhostTime(FieldValue, Variable): + """ + Список имен прочих репозиториев + """ + type = "list" + source_variable = "cl_update_binhost_data" + column = 3 + + class VariableClUpdateLastTimestamp(ReadonlyVariable): """ Текущий timestamp @@ -424,15 +373,6 @@ class VariableClUpdateLastTimestamp(ReadonlyVariable): return ini.getVar('system', 'last_update') or "0" -class VariableClUpdateBinhostTime(FieldValue, ReadonlyVariable): - """ - Список имен прочих репозиториев - """ - type = "list" - source_variable = "cl_update_binhost_data" - column = 3 - - class VariableClUpdateBranchName(Variable): """ Список доступных репозиторием @@ -586,6 +526,29 @@ class VariableClUpdateEixupdateForce(Variable): ("auto", _("If needed"))] +class VariableClUpdateGpgForce(Variable): + """ + Принудительное действие с eix-update + """ + type = "choice" + value = "auto" + opt = ["--check-sign"] + syntax = "--{choice}-check-sign" + metavalue = "MODE" + # untrusted = True + + def init(self): + self.help = ("'force' - " + _("force check Package signature") + + ",\n'skip' - " + _("skip check Package signature") + + ",\n'auto' - " + _("check signature if system has public keys")) + self.label = _("Check Package signature") + + def choice(self): + return [("force", _("Force")), + ("skip", _("Skip")), + ("auto", _("If has passibility"))] + + class VariableClUpdateOtherSet(Variable): """ Обновить остальные оверлеи @@ -811,9 +774,6 @@ class VariableClUpdateProfileDependData(ReadonlyTableVariable): def get(self, hr=HumanReadable.No): dv = self.Get(self.datavars) - # TODO: неиспользуемая переменная возможно - # испольуется для инициализации - # url = self.Get('cl_update_profile_url').lower() if dv: if hr == HumanReadable.Yes: return reversed(zip([x.capitalize() for x in dv.Get('cl_update_rep_name')], @@ -1076,17 +1036,6 @@ class VariableClUpdateProfileBranch(Variable): self.label = _("Repository branch") self.help = _("set the repository branch") - def check(self, value): - pass - ## TODO: проверка ветки - # try: - # url = self.Get('cl_update_profile_url') - # self.Get('cl_update_profile_storage').get_profiles(url, value) - # except GitError as e: - - -# raise VariableError(str(e)) - class VariableClProfileRepository(ReadonlyVariable): """ @@ -1736,6 +1685,12 @@ class VariableClUpdatePackageCache(ReadonlyVariable): host, parsed_url.path.lstrip("/"), "Packages") +class VariableClUpdatePackageCacheSign(ReadonlyVariable): + """ + Файл с подписью Packages находящийся в кэше + """ + value_format = "{update.cl_update_package_cache}.asc" + class VariableClUpdatePackageCacheSet(Variable): """ @@ -1745,7 +1700,9 @@ class VariableClUpdatePackageCacheSet(Variable): def get(self): packages_fn = self.Get('cl_update_package_cache') - if not path.exists(packages_fn): + packages_asc_fn = self.Get('cl_update_package_cache_sign') + if (not path.exists(packages_fn) or + not path.exists(packages_asc_fn)): return "on" pi = PackagesIndex(readFile(packages_fn)) try: @@ -1800,3 +1757,42 @@ class VariableClUpdateSshkeyPath(Variable): Путь до ssh ключа """ value = "/var/lib/calculate/id_rsa" + +class VariableClUpdateBadSignSet(Variable): + """ + Ошибка проверки подписи Packages + """ + type = Variable.Types.Boolean + value = Variable.Off + +class VariableClUpdateGpgKeys(Variable): + """ + Список ключей для проверки подписи Packages + """ + type = Variable.Types.List + value = ["/usr/share/openpgp-keys/calculate-release.asc"] + +class VariableClUpdateGpg(ReadonlyVariable): + """ + Объект проверки подписи + """ + type = Variable.Types.Object + value = "" + + def close(self): + val = self.Get() + if val: + val.close() + +class VariableClUpdateUseDowngradeSet(ReadonlyVariable): + """ + Можно использовать downgrade binhost + """ + type = Variable.Types.Boolean + + def get(self): + """ + Выполняется переход с unstable ветки на stable + """ + return (self.GetBool("cl_update_binhost_stable_opt_set") and + not self.GetBool("cl_update_binhost_stable_opt_set")) diff --git a/pym/update/wsdl_update.py b/pym/update/wsdl_update.py index 472c9c7..0d38bdf 100644 --- a/pym/update/wsdl_update.py +++ b/pym/update/wsdl_update.py @@ -78,6 +78,7 @@ class Wsdl(WsdlBase): 'cl_update_check_rep_set', 'cl_update_force_fix_set', 'cl_update_world', + 'cl_update_gpg_force', 'cl_update_egencache_force', 'cl_update_eixupdate_force', 'cl_update_skip_rb_set',