Добавлено логгирование обновления списка бинхостов

master3.4 3.4.5.5
parent 3adbb14780
commit b6759a7040

@ -19,11 +19,13 @@ import sys
import re
from os import path
from calculate.lib.cl_template import SystemIni
from calculate.lib.cl_log import log
from calculate.lib.datavars import (Variable, VariableError,
ReadonlyVariable, ReadonlyTableVariable,
TableVariable, FieldValue,
HumanReadable,
SimpleDataVars, DataVarsError)
from calculate.lib.utils.binhosts import Binhosts
from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin
from calculate.lib.configparser import ConfigParser
@ -37,7 +39,6 @@ from ..profile import (RepositoryStorageSet, DEFAULT_BRANCH,
from calculate.lib.variables import linux as lib_linux
from calculate.lib.variables import env
from calculate.update.update_info import UpdateInfo
import urllib2
import time
_ = lambda x: x
@ -281,105 +282,70 @@ class VariableClUpdateBinhostData(ReadonlyTableVariable):
"cl_update_binhost_timestamp",
"cl_update_binhost_time"]
def check_binhost(self, binhost):
revision_files = [path.join(binhost, x)
for x in self.Get('cl_update_binhost_revision_path')]
timeout = self.GetInteger('cl_update_binhost_timeout')
try:
data = None
for fn in revision_files:
if data is None:
data = urllib2.urlopen(fn, timeout=timeout).read()
elif data != urllib2.urlopen(fn, timeout=timeout).read():
return ""
return data
except urllib2.URLError:
return ""
re_revison = re.compile("\w+=(\w+)")
def binhost_key(self, data):
host, t = data
try:
cp = ConfigParser()
cp.read_string(data.decode('utf-8'))
data = sum(int(x) for x in cp['vcs'].values())
except (TypeError, KeyError) as e:
data = 0
return (1 if int(time) >= 0 else 0,
data,
-int(time))
def get_timestamp(self, binhost):
"""
Возвращает таймстам полученный от сервера, время доступа,
актуальность (не более 5 суток)
"""
DAY = 60 * 60 * 24
timeout = self.GetInteger('cl_update_binhost_timeout')
timestamp_file = path.join(binhost,
self.Get('cl_update_binhost_timestamp_path'))
last_ts = self.Get('cl_update_last_timestamp')
try:
t = time.time()
data = urllib2.urlopen(timestamp_file,
timeout=timeout).read().strip()
if (data.isdigit() and last_ts.isdigit() and
int(data) >= int(last_ts)):
return (data, int((time.time() - t) * 1000),
t - int(data) < 5 * DAY)
except urllib2.URLError as e:
pass
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
pass
return "", -1, False
def get_logger(self, stub=False):
class StubLogger(object):
def info(self, message):
pass
if not stub and (not self.Get('cl_ebuild_phase')):
# проверка на chroot_path
return log("binhost-scan.log",
filename=pathJoin(
self.Get('cl_chroot_path'),
"/var/log/calculate/binhost-scan.log"),
formatter="%(message)s")
else:
return StubLogger()
def get(self, hr=HumanReadable.No):
DAY = 60 * 60 * 24
last_ts = self.Get('cl_update_last_timestamp')
binhosts_data = Binhosts(
self.GetInteger('cl_update_binhost_timeout'),
self.Get('cl_update_binhost_revision_path'),
self.Get('cl_update_binhost_timestamp_path'),
last_ts,
self.Get('cl_update_binhost_list')
)
binhost = self.Get('cl_update_binhost')
recheck = self.GetBool('cl_update_binhost_recheck_set')
cur_t = time.time()
def generate_by_timestamp():
for host in self.Get('cl_update_binhost_list'):
if host:
ts_content, duration, good = self.get_timestamp(host)
if ts_content:
yield host, ts_content, str(duration), good
if not recheck and binhost:
ts, t, good = self.get_timestamp(binhost)
ts, t, good, downgrade = binhosts_data.get_timestamp(binhost)
# условие актуальности текущего сервера
if ts and cur_t - int(ts) < 5 * DAY:
data = self.check_binhost(binhost)
if ts and good and not downgrade:
data = binhosts_data.check_binhost(binhost)
if data:
return [[binhost, data, ts, str(t)]]
for host, ts, t, good in sorted(
generate_by_timestamp(),
# критерий сортировки между серверами
# актуальные сортируются по скорости
# неактульные по timestamp
# актуальность - 5 дней
reverse=True,
key=lambda x: ( # приоритетны актуальные (не более 5 дней)
x[3],
# приоритет для неактуальных
# самое свежее
0 if x[3] else int(x[1]),
# приоритет для неактуальных
# самое быстрое
0 if x[3] else -int(x[2]),
# приоритет для актуальных
# самое быстрое
-int(x[2]) if x[3] else 0,
# самое свежее
int(x[1]) if x[3] else 0)):
data = self.check_binhost(host)
if data:
return [[host, data, ts, str(t)]]
return [[]]
logger = self.get_logger(stub=binhosts_data.is_cache())
ret_val = None
# TODO: важно еще сверу коммент
# отладочная строка
# необходимо разобраться почему calculate-builder вызывает
# заполнение несколько раз!!!!!!!!!!!!
logger.info("Started scan on: {date}, current timestamp: {ts}".format(
date=time.ctime(), ts=last_ts))
for host, ts, t, good, downgrade in binhosts_data.get_sorted():
if ret_val is None:
data = binhosts_data.check_binhost(host)
else:
data = ""
if not good:
reason = "OUTDATED"
elif downgrade:
reason = "SKIP"
elif not data and ret_val is None:
reason = "UPDATING"
else:
reason = ""
logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format(
host=host, speed=float(t) / 1000.0, timestamp=ts,
reason=reason))
if ret_val is None and data and not downgrade:
ret_val = [[host, data, ts, str(t)]]
return ret_val or [[]]
class VariableClUpdateBinhostRecheckSet(Variable):
@ -1313,6 +1279,7 @@ class DataVarsUpdateProfile(SimpleDataVars):
env.VariableClTemplatePath(),
env.VariableClEmergeConfig(systemRoot=chroot_path),
env.VariableClFeatures(),
env.VariableClEbuildPhase(),
VariableClUpdateRepData(section="update"),
VariableClUpdateRepPath(section="update"),
VariableClUpdateRepRev(section="update"),

Caricamento…
Annulla
Salva