Добавлено логгирование обновления списка бинхостов

master3.4 3.4.5.5
parent 3adbb14780
commit b6759a7040

@ -19,11 +19,13 @@ import sys
import re import re
from os import path from os import path
from calculate.lib.cl_template import SystemIni from calculate.lib.cl_template import SystemIni
from calculate.lib.cl_log import log
from calculate.lib.datavars import (Variable, VariableError, from calculate.lib.datavars import (Variable, VariableError,
ReadonlyVariable, ReadonlyTableVariable, ReadonlyVariable, ReadonlyTableVariable,
TableVariable, FieldValue, TableVariable, FieldValue,
HumanReadable, HumanReadable,
SimpleDataVars, DataVarsError) SimpleDataVars, DataVarsError)
from calculate.lib.utils.binhosts import Binhosts
from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin
from calculate.lib.configparser import ConfigParser from calculate.lib.configparser import ConfigParser
@ -37,7 +39,6 @@ from ..profile import (RepositoryStorageSet, DEFAULT_BRANCH,
from calculate.lib.variables import linux as lib_linux from calculate.lib.variables import linux as lib_linux
from calculate.lib.variables import env from calculate.lib.variables import env
from calculate.update.update_info import UpdateInfo from calculate.update.update_info import UpdateInfo
import urllib2
import time import time
_ = lambda x: x _ = lambda x: x
@ -281,105 +282,70 @@ class VariableClUpdateBinhostData(ReadonlyTableVariable):
"cl_update_binhost_timestamp", "cl_update_binhost_timestamp",
"cl_update_binhost_time"] "cl_update_binhost_time"]
def check_binhost(self, binhost): def get_logger(self, stub=False):
revision_files = [path.join(binhost, x) class StubLogger(object):
for x in self.Get('cl_update_binhost_revision_path')] def info(self, message):
timeout = self.GetInteger('cl_update_binhost_timeout') pass
try:
data = None if not stub and (not self.Get('cl_ebuild_phase')):
for fn in revision_files: # проверка на chroot_path
if data is None: return log("binhost-scan.log",
data = urllib2.urlopen(fn, timeout=timeout).read() filename=pathJoin(
elif data != urllib2.urlopen(fn, timeout=timeout).read(): self.Get('cl_chroot_path'),
return "" "/var/log/calculate/binhost-scan.log"),
return data formatter="%(message)s")
except urllib2.URLError: else:
return "" return StubLogger()
re_revison = re.compile("\w+=(\w+)")
def binhost_key(self, data):
host, t = data
try:
cp = ConfigParser()
cp.read_string(data.decode('utf-8'))
data = sum(int(x) for x in cp['vcs'].values())
except (TypeError, KeyError) as e:
data = 0
return (1 if int(time) >= 0 else 0,
data,
-int(time))
def get_timestamp(self, binhost):
"""
Возвращает таймстам полученный от сервера, время доступа,
актуальность (не более 5 суток)
"""
DAY = 60 * 60 * 24
timeout = self.GetInteger('cl_update_binhost_timeout')
timestamp_file = path.join(binhost,
self.Get('cl_update_binhost_timestamp_path'))
last_ts = self.Get('cl_update_last_timestamp')
try:
t = time.time()
data = urllib2.urlopen(timestamp_file,
timeout=timeout).read().strip()
if (data.isdigit() and last_ts.isdigit() and
int(data) >= int(last_ts)):
return (data, int((time.time() - t) * 1000),
t - int(data) < 5 * DAY)
except urllib2.URLError as e:
pass
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
pass
return "", -1, False
def get(self, hr=HumanReadable.No): def get(self, hr=HumanReadable.No):
DAY = 60 * 60 * 24 last_ts = self.Get('cl_update_last_timestamp')
binhosts_data = Binhosts(
self.GetInteger('cl_update_binhost_timeout'),
self.Get('cl_update_binhost_revision_path'),
self.Get('cl_update_binhost_timestamp_path'),
last_ts,
self.Get('cl_update_binhost_list')
)
binhost = self.Get('cl_update_binhost') binhost = self.Get('cl_update_binhost')
recheck = self.GetBool('cl_update_binhost_recheck_set') recheck = self.GetBool('cl_update_binhost_recheck_set')
cur_t = time.time()
def generate_by_timestamp():
for host in self.Get('cl_update_binhost_list'):
if host:
ts_content, duration, good = self.get_timestamp(host)
if ts_content:
yield host, ts_content, str(duration), good
if not recheck and binhost: if not recheck and binhost:
ts, t, good = self.get_timestamp(binhost) ts, t, good, downgrade = binhosts_data.get_timestamp(binhost)
# условие актуальности текущего сервера # условие актуальности текущего сервера
if ts and cur_t - int(ts) < 5 * DAY: if ts and good and not downgrade:
data = self.check_binhost(binhost) data = binhosts_data.check_binhost(binhost)
if data: if data:
return [[binhost, data, ts, str(t)]] return [[binhost, data, ts, str(t)]]
for host, ts, t, good in sorted(
generate_by_timestamp(), logger = self.get_logger(stub=binhosts_data.is_cache())
# критерий сортировки между серверами ret_val = None
# актуальные сортируются по скорости # TODO: важно еще сверу коммент
# неактульные по timestamp # отладочная строка
# актуальность - 5 дней # необходимо разобраться почему calculate-builder вызывает
reverse=True, # заполнение несколько раз!!!!!!!!!!!!
key=lambda x: ( # приоритетны актуальные (не более 5 дней) logger.info("Started scan on: {date}, current timestamp: {ts}".format(
x[3], date=time.ctime(), ts=last_ts))
# приоритет для неактуальных for host, ts, t, good, downgrade in binhosts_data.get_sorted():
# самое свежее if ret_val is None:
0 if x[3] else int(x[1]), data = binhosts_data.check_binhost(host)
# приоритет для неактуальных else:
# самое быстрое data = ""
0 if x[3] else -int(x[2]),
# приоритет для актуальных if not good:
# самое быстрое reason = "OUTDATED"
-int(x[2]) if x[3] else 0, elif downgrade:
# самое свежее reason = "SKIP"
int(x[1]) if x[3] else 0)): elif not data and ret_val is None:
data = self.check_binhost(host) reason = "UPDATING"
if data: else:
return [[host, data, ts, str(t)]] reason = ""
return [[]]
logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format(
host=host, speed=float(t) / 1000.0, timestamp=ts,
reason=reason))
if ret_val is None and data and not downgrade:
ret_val = [[host, data, ts, str(t)]]
return ret_val or [[]]
class VariableClUpdateBinhostRecheckSet(Variable): class VariableClUpdateBinhostRecheckSet(Variable):
@ -1313,6 +1279,7 @@ class DataVarsUpdateProfile(SimpleDataVars):
env.VariableClTemplatePath(), env.VariableClTemplatePath(),
env.VariableClEmergeConfig(systemRoot=chroot_path), env.VariableClEmergeConfig(systemRoot=chroot_path),
env.VariableClFeatures(), env.VariableClFeatures(),
env.VariableClEbuildPhase(),
VariableClUpdateRepData(section="update"), VariableClUpdateRepData(section="update"),
VariableClUpdateRepPath(section="update"), VariableClUpdateRepPath(section="update"),
VariableClUpdateRepRev(section="update"), VariableClUpdateRepRev(section="update"),

Loading…
Cancel
Save