From b6759a7040f61050eb88ae19cf10ed7b9da124cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B8=D1=80=D0=B5=D1=86=D0=BA=D0=B8=D0=B9=20=D0=9C?= =?UTF-8?q?=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= Date: Tue, 22 Dec 2015 17:51:49 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=BE=20=D0=BB=D0=BE=D0=B3=D0=B3=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=B0?= =?UTF-8?q?=20=D0=B1=D0=B8=D0=BD=D1=85=D0=BE=D1=81=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/update/variables/update.py | 149 +++++++++++++-------------------- 1 file changed, 58 insertions(+), 91 deletions(-) diff --git a/pym/update/variables/update.py b/pym/update/variables/update.py index 4c90fb1..e3d8636 100644 --- a/pym/update/variables/update.py +++ b/pym/update/variables/update.py @@ -19,11 +19,13 @@ import sys import re from os import path from calculate.lib.cl_template import SystemIni +from calculate.lib.cl_log import log from calculate.lib.datavars import (Variable, VariableError, ReadonlyVariable, ReadonlyTableVariable, TableVariable, FieldValue, HumanReadable, SimpleDataVars, DataVarsError) +from calculate.lib.utils.binhosts import Binhosts from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin from calculate.lib.configparser import ConfigParser @@ -37,7 +39,6 @@ from ..profile import (RepositoryStorageSet, DEFAULT_BRANCH, from calculate.lib.variables import linux as lib_linux from calculate.lib.variables import env from calculate.update.update_info import UpdateInfo -import urllib2 import time _ = lambda x: x @@ -281,105 +282,70 @@ class VariableClUpdateBinhostData(ReadonlyTableVariable): "cl_update_binhost_timestamp", "cl_update_binhost_time"] - def check_binhost(self, binhost): - revision_files = [path.join(binhost, x) - for x in self.Get('cl_update_binhost_revision_path')] - timeout = self.GetInteger('cl_update_binhost_timeout') - try: - data = None - for fn in revision_files: - if data is None: - data = urllib2.urlopen(fn, timeout=timeout).read() - elif data != urllib2.urlopen(fn, timeout=timeout).read(): - return "" - return data - except urllib2.URLError: - return "" - - re_revison = re.compile("\w+=(\w+)") - - def binhost_key(self, data): - host, t = data - try: - cp = ConfigParser() - cp.read_string(data.decode('utf-8')) - data = sum(int(x) for x in cp['vcs'].values()) - except (TypeError, KeyError) as e: - data = 0 - return (1 if int(time) >= 0 else 0, - data, - -int(time)) - - def get_timestamp(self, binhost): - """ - Возвращает таймстам полученный от сервера, время доступа, - актуальность (не более 5 суток) - """ - DAY = 60 * 60 * 24 - timeout = self.GetInteger('cl_update_binhost_timeout') - timestamp_file = path.join(binhost, - self.Get('cl_update_binhost_timestamp_path')) - last_ts = self.Get('cl_update_last_timestamp') - try: - t = time.time() - data = urllib2.urlopen(timestamp_file, - timeout=timeout).read().strip() - if (data.isdigit() and last_ts.isdigit() and - int(data) >= int(last_ts)): - return (data, int((time.time() - t) * 1000), - t - int(data) < 5 * DAY) - except urllib2.URLError as e: - pass - except BaseException as e: - if isinstance(e, KeyboardInterrupt): - raise - pass - return "", -1, False + def get_logger(self, stub=False): + class StubLogger(object): + def info(self, message): + pass + + if not stub and (not self.Get('cl_ebuild_phase')): + # проверка на chroot_path + return log("binhost-scan.log", + filename=pathJoin( + self.Get('cl_chroot_path'), + "/var/log/calculate/binhost-scan.log"), + formatter="%(message)s") + else: + return StubLogger() def get(self, hr=HumanReadable.No): - DAY = 60 * 60 * 24 + last_ts = self.Get('cl_update_last_timestamp') + binhosts_data = Binhosts( + self.GetInteger('cl_update_binhost_timeout'), + self.Get('cl_update_binhost_revision_path'), + self.Get('cl_update_binhost_timestamp_path'), + last_ts, + self.Get('cl_update_binhost_list') + ) binhost = self.Get('cl_update_binhost') recheck = self.GetBool('cl_update_binhost_recheck_set') - cur_t = time.time() - - def generate_by_timestamp(): - for host in self.Get('cl_update_binhost_list'): - if host: - ts_content, duration, good = self.get_timestamp(host) - if ts_content: - yield host, ts_content, str(duration), good if not recheck and binhost: - ts, t, good = self.get_timestamp(binhost) + ts, t, good, downgrade = binhosts_data.get_timestamp(binhost) # условие актуальности текущего сервера - if ts and cur_t - int(ts) < 5 * DAY: - data = self.check_binhost(binhost) + if ts and good and not downgrade: + data = binhosts_data.check_binhost(binhost) if data: return [[binhost, data, ts, str(t)]] - for host, ts, t, good in sorted( - generate_by_timestamp(), - # критерий сортировки между серверами - # актуальные сортируются по скорости - # неактульные по timestamp - # актуальность - 5 дней - reverse=True, - key=lambda x: ( # приоритетны актуальные (не более 5 дней) - x[3], - # приоритет для неактуальных - # самое свежее - 0 if x[3] else int(x[1]), - # приоритет для неактуальных - # самое быстрое - 0 if x[3] else -int(x[2]), - # приоритет для актуальных - # самое быстрое - -int(x[2]) if x[3] else 0, - # самое свежее - int(x[1]) if x[3] else 0)): - data = self.check_binhost(host) - if data: - return [[host, data, ts, str(t)]] - return [[]] + + logger = self.get_logger(stub=binhosts_data.is_cache()) + ret_val = None + # TODO: важно еще сверу коммент + # отладочная строка + # необходимо разобраться почему calculate-builder вызывает + # заполнение несколько раз!!!!!!!!!!!! + logger.info("Started scan on: {date}, current timestamp: {ts}".format( + date=time.ctime(), ts=last_ts)) + for host, ts, t, good, downgrade in binhosts_data.get_sorted(): + if ret_val is None: + data = binhosts_data.check_binhost(host) + else: + data = "" + + if not good: + reason = "OUTDATED" + elif downgrade: + reason = "SKIP" + elif not data and ret_val is None: + reason = "UPDATING" + else: + reason = "" + + logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format( + host=host, speed=float(t) / 1000.0, timestamp=ts, + reason=reason)) + if ret_val is None and data and not downgrade: + ret_val = [[host, data, ts, str(t)]] + return ret_val or [[]] class VariableClUpdateBinhostRecheckSet(Variable): @@ -1313,6 +1279,7 @@ class DataVarsUpdateProfile(SimpleDataVars): env.VariableClTemplatePath(), env.VariableClEmergeConfig(systemRoot=chroot_path), env.VariableClFeatures(), + env.VariableClEbuildPhase(), VariableClUpdateRepData(section="update"), VariableClUpdateRepPath(section="update"), VariableClUpdateRepRev(section="update"),