|
|
@ -19,11 +19,13 @@ import sys
|
|
|
|
import re
|
|
|
|
import re
|
|
|
|
from os import path
|
|
|
|
from os import path
|
|
|
|
from calculate.lib.cl_template import SystemIni
|
|
|
|
from calculate.lib.cl_template import SystemIni
|
|
|
|
|
|
|
|
from calculate.lib.cl_log import log
|
|
|
|
from calculate.lib.datavars import (Variable, VariableError,
|
|
|
|
from calculate.lib.datavars import (Variable, VariableError,
|
|
|
|
ReadonlyVariable, ReadonlyTableVariable,
|
|
|
|
ReadonlyVariable, ReadonlyTableVariable,
|
|
|
|
TableVariable, FieldValue,
|
|
|
|
TableVariable, FieldValue,
|
|
|
|
HumanReadable,
|
|
|
|
HumanReadable,
|
|
|
|
SimpleDataVars, DataVarsError)
|
|
|
|
SimpleDataVars, DataVarsError)
|
|
|
|
|
|
|
|
from calculate.lib.utils.binhosts import Binhosts
|
|
|
|
from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin
|
|
|
|
from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin
|
|
|
|
|
|
|
|
|
|
|
|
from calculate.lib.configparser import ConfigParser
|
|
|
|
from calculate.lib.configparser import ConfigParser
|
|
|
@ -37,7 +39,6 @@ from ..profile import (RepositoryStorageSet, DEFAULT_BRANCH,
|
|
|
|
from calculate.lib.variables import linux as lib_linux
|
|
|
|
from calculate.lib.variables import linux as lib_linux
|
|
|
|
from calculate.lib.variables import env
|
|
|
|
from calculate.lib.variables import env
|
|
|
|
from calculate.update.update_info import UpdateInfo
|
|
|
|
from calculate.update.update_info import UpdateInfo
|
|
|
|
import urllib2
|
|
|
|
|
|
|
|
import time
|
|
|
|
import time
|
|
|
|
|
|
|
|
|
|
|
|
_ = lambda x: x
|
|
|
|
_ = lambda x: x
|
|
|
@ -281,105 +282,70 @@ class VariableClUpdateBinhostData(ReadonlyTableVariable):
|
|
|
|
"cl_update_binhost_timestamp",
|
|
|
|
"cl_update_binhost_timestamp",
|
|
|
|
"cl_update_binhost_time"]
|
|
|
|
"cl_update_binhost_time"]
|
|
|
|
|
|
|
|
|
|
|
|
def check_binhost(self, binhost):
|
|
|
|
def get_logger(self, stub=False):
|
|
|
|
revision_files = [path.join(binhost, x)
|
|
|
|
class StubLogger(object):
|
|
|
|
for x in self.Get('cl_update_binhost_revision_path')]
|
|
|
|
def info(self, message):
|
|
|
|
timeout = self.GetInteger('cl_update_binhost_timeout')
|
|
|
|
pass
|
|
|
|
try:
|
|
|
|
|
|
|
|
data = None
|
|
|
|
if not stub and (not self.Get('cl_ebuild_phase')):
|
|
|
|
for fn in revision_files:
|
|
|
|
# проверка на chroot_path
|
|
|
|
if data is None:
|
|
|
|
return log("binhost-scan.log",
|
|
|
|
data = urllib2.urlopen(fn, timeout=timeout).read()
|
|
|
|
filename=pathJoin(
|
|
|
|
elif data != urllib2.urlopen(fn, timeout=timeout).read():
|
|
|
|
self.Get('cl_chroot_path'),
|
|
|
|
return ""
|
|
|
|
"/var/log/calculate/binhost-scan.log"),
|
|
|
|
return data
|
|
|
|
formatter="%(message)s")
|
|
|
|
except urllib2.URLError:
|
|
|
|
else:
|
|
|
|
return ""
|
|
|
|
return StubLogger()
|
|
|
|
|
|
|
|
|
|
|
|
re_revison = re.compile("\w+=(\w+)")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def binhost_key(self, data):
|
|
|
|
|
|
|
|
host, t = data
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
cp = ConfigParser()
|
|
|
|
|
|
|
|
cp.read_string(data.decode('utf-8'))
|
|
|
|
|
|
|
|
data = sum(int(x) for x in cp['vcs'].values())
|
|
|
|
|
|
|
|
except (TypeError, KeyError) as e:
|
|
|
|
|
|
|
|
data = 0
|
|
|
|
|
|
|
|
return (1 if int(time) >= 0 else 0,
|
|
|
|
|
|
|
|
data,
|
|
|
|
|
|
|
|
-int(time))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_timestamp(self, binhost):
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
Возвращает таймстам полученный от сервера, время доступа,
|
|
|
|
|
|
|
|
актуальность (не более 5 суток)
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
DAY = 60 * 60 * 24
|
|
|
|
|
|
|
|
timeout = self.GetInteger('cl_update_binhost_timeout')
|
|
|
|
|
|
|
|
timestamp_file = path.join(binhost,
|
|
|
|
|
|
|
|
self.Get('cl_update_binhost_timestamp_path'))
|
|
|
|
|
|
|
|
last_ts = self.Get('cl_update_last_timestamp')
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
t = time.time()
|
|
|
|
|
|
|
|
data = urllib2.urlopen(timestamp_file,
|
|
|
|
|
|
|
|
timeout=timeout).read().strip()
|
|
|
|
|
|
|
|
if (data.isdigit() and last_ts.isdigit() and
|
|
|
|
|
|
|
|
int(data) >= int(last_ts)):
|
|
|
|
|
|
|
|
return (data, int((time.time() - t) * 1000),
|
|
|
|
|
|
|
|
t - int(data) < 5 * DAY)
|
|
|
|
|
|
|
|
except urllib2.URLError as e:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
except BaseException as e:
|
|
|
|
|
|
|
|
if isinstance(e, KeyboardInterrupt):
|
|
|
|
|
|
|
|
raise
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
return "", -1, False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get(self, hr=HumanReadable.No):
|
|
|
|
def get(self, hr=HumanReadable.No):
|
|
|
|
DAY = 60 * 60 * 24
|
|
|
|
last_ts = self.Get('cl_update_last_timestamp')
|
|
|
|
|
|
|
|
binhosts_data = Binhosts(
|
|
|
|
|
|
|
|
self.GetInteger('cl_update_binhost_timeout'),
|
|
|
|
|
|
|
|
self.Get('cl_update_binhost_revision_path'),
|
|
|
|
|
|
|
|
self.Get('cl_update_binhost_timestamp_path'),
|
|
|
|
|
|
|
|
last_ts,
|
|
|
|
|
|
|
|
self.Get('cl_update_binhost_list')
|
|
|
|
|
|
|
|
)
|
|
|
|
binhost = self.Get('cl_update_binhost')
|
|
|
|
binhost = self.Get('cl_update_binhost')
|
|
|
|
recheck = self.GetBool('cl_update_binhost_recheck_set')
|
|
|
|
recheck = self.GetBool('cl_update_binhost_recheck_set')
|
|
|
|
cur_t = time.time()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def generate_by_timestamp():
|
|
|
|
|
|
|
|
for host in self.Get('cl_update_binhost_list'):
|
|
|
|
|
|
|
|
if host:
|
|
|
|
|
|
|
|
ts_content, duration, good = self.get_timestamp(host)
|
|
|
|
|
|
|
|
if ts_content:
|
|
|
|
|
|
|
|
yield host, ts_content, str(duration), good
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if not recheck and binhost:
|
|
|
|
if not recheck and binhost:
|
|
|
|
ts, t, good = self.get_timestamp(binhost)
|
|
|
|
ts, t, good, downgrade = binhosts_data.get_timestamp(binhost)
|
|
|
|
# условие актуальности текущего сервера
|
|
|
|
# условие актуальности текущего сервера
|
|
|
|
if ts and cur_t - int(ts) < 5 * DAY:
|
|
|
|
if ts and good and not downgrade:
|
|
|
|
data = self.check_binhost(binhost)
|
|
|
|
data = binhosts_data.check_binhost(binhost)
|
|
|
|
if data:
|
|
|
|
if data:
|
|
|
|
return [[binhost, data, ts, str(t)]]
|
|
|
|
return [[binhost, data, ts, str(t)]]
|
|
|
|
for host, ts, t, good in sorted(
|
|
|
|
|
|
|
|
generate_by_timestamp(),
|
|
|
|
logger = self.get_logger(stub=binhosts_data.is_cache())
|
|
|
|
# критерий сортировки между серверами
|
|
|
|
ret_val = None
|
|
|
|
# актуальные сортируются по скорости
|
|
|
|
# TODO: важно еще сверу коммент
|
|
|
|
# неактульные по timestamp
|
|
|
|
# отладочная строка
|
|
|
|
# актуальность - 5 дней
|
|
|
|
# необходимо разобраться почему calculate-builder вызывает
|
|
|
|
reverse=True,
|
|
|
|
# заполнение несколько раз!!!!!!!!!!!!
|
|
|
|
key=lambda x: ( # приоритетны актуальные (не более 5 дней)
|
|
|
|
logger.info("Started scan on: {date}, current timestamp: {ts}".format(
|
|
|
|
x[3],
|
|
|
|
date=time.ctime(), ts=last_ts))
|
|
|
|
# приоритет для неактуальных
|
|
|
|
for host, ts, t, good, downgrade in binhosts_data.get_sorted():
|
|
|
|
# самое свежее
|
|
|
|
if ret_val is None:
|
|
|
|
0 if x[3] else int(x[1]),
|
|
|
|
data = binhosts_data.check_binhost(host)
|
|
|
|
# приоритет для неактуальных
|
|
|
|
else:
|
|
|
|
# самое быстрое
|
|
|
|
data = ""
|
|
|
|
0 if x[3] else -int(x[2]),
|
|
|
|
|
|
|
|
# приоритет для актуальных
|
|
|
|
if not good:
|
|
|
|
# самое быстрое
|
|
|
|
reason = "OUTDATED"
|
|
|
|
-int(x[2]) if x[3] else 0,
|
|
|
|
elif downgrade:
|
|
|
|
# самое свежее
|
|
|
|
reason = "SKIP"
|
|
|
|
int(x[1]) if x[3] else 0)):
|
|
|
|
elif not data and ret_val is None:
|
|
|
|
data = self.check_binhost(host)
|
|
|
|
reason = "UPDATING"
|
|
|
|
if data:
|
|
|
|
else:
|
|
|
|
return [[host, data, ts, str(t)]]
|
|
|
|
reason = ""
|
|
|
|
return [[]]
|
|
|
|
|
|
|
|
|
|
|
|
logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format(
|
|
|
|
|
|
|
|
host=host, speed=float(t) / 1000.0, timestamp=ts,
|
|
|
|
|
|
|
|
reason=reason))
|
|
|
|
|
|
|
|
if ret_val is None and data and not downgrade:
|
|
|
|
|
|
|
|
ret_val = [[host, data, ts, str(t)]]
|
|
|
|
|
|
|
|
return ret_val or [[]]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class VariableClUpdateBinhostRecheckSet(Variable):
|
|
|
|
class VariableClUpdateBinhostRecheckSet(Variable):
|
|
|
@ -1313,6 +1279,7 @@ class DataVarsUpdateProfile(SimpleDataVars):
|
|
|
|
env.VariableClTemplatePath(),
|
|
|
|
env.VariableClTemplatePath(),
|
|
|
|
env.VariableClEmergeConfig(systemRoot=chroot_path),
|
|
|
|
env.VariableClEmergeConfig(systemRoot=chroot_path),
|
|
|
|
env.VariableClFeatures(),
|
|
|
|
env.VariableClFeatures(),
|
|
|
|
|
|
|
|
env.VariableClEbuildPhase(),
|
|
|
|
VariableClUpdateRepData(section="update"),
|
|
|
|
VariableClUpdateRepData(section="update"),
|
|
|
|
VariableClUpdateRepPath(section="update"),
|
|
|
|
VariableClUpdateRepPath(section="update"),
|
|
|
|
VariableClUpdateRepRev(section="update"),
|
|
|
|
VariableClUpdateRepRev(section="update"),
|
|
|
|