You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
262 lines
9.3 KiB
262 lines
9.3 KiB
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2015-2016 Mir Calculate. http://www.calculate-linux.org
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import sys
|
|
import urllib2
|
|
import re
|
|
import time
|
|
import os
|
|
from os import path
|
|
from calculate.lib.configparser import ConfigParserCaseSens, Error as CPError
|
|
from calculate.lib.utils.tools import SingletonParam
|
|
from .files import writeFile, xz
|
|
from collections import MutableMapping
|
|
|
|
_ = lambda x: x
|
|
from calculate.lib.cl_lang import setLocalTranslate
|
|
|
|
setLocalTranslate('cl_lib3', sys.modules[__name__])
|
|
|
|
MINUTES = 60
|
|
HOURS = 60 * MINUTES
|
|
DAYS = 24 * HOURS
|
|
|
|
def _urlopen(fn, timeout=None):
|
|
"""
|
|
Получить URL, без прокси
|
|
"""
|
|
proxy_handler = urllib2.ProxyHandler({})
|
|
opener = urllib2.build_opener(proxy_handler)
|
|
if timeout is not None:
|
|
return opener.open(fn, timeout=timeout)
|
|
else:
|
|
return opener.open(fn)
|
|
|
|
|
|
class BinhostsBase(object):
|
|
class BinhostStatus:
|
|
Success = 0
|
|
EnvNotFound = 1
|
|
Updating = 2
|
|
BadEnv = 3
|
|
UnknownError = 4
|
|
|
|
def __init__(self, timeout, revision_path, ts_path, last_ts, binhost_list,
|
|
arch=None, skip_timestamp=False):
|
|
self.timeout = int(timeout)
|
|
self.revision_path = revision_path
|
|
self.ts_path = ts_path
|
|
self.last_ts = last_ts
|
|
self.binhost_list = binhost_list
|
|
self.data = None
|
|
self.arch = arch
|
|
self.skip_timestamp = skip_timestamp
|
|
self.binhosts_data = {}
|
|
|
|
def check_package_timestamp(self, fn, timestamp):
|
|
for i, line in enumerate(
|
|
_urlopen(fn, timeout=self.timeout)):
|
|
if line.startswith("TIMESTAMP"):
|
|
return str(timestamp) == line.rpartition(":")[2].strip()
|
|
if i > 50:
|
|
break
|
|
return False
|
|
|
|
def check_binhost(self, binhost):
|
|
status = self.BinhostStatus.Success
|
|
if binhost not in self.binhosts_data:
|
|
revision_files = [path.join(binhost, x)
|
|
for x in self.revision_path]
|
|
data = ""
|
|
try:
|
|
for fn in revision_files:
|
|
base_dn = path.dirname(fn)
|
|
data = _urlopen(fn, timeout=self.timeout).read()
|
|
cp = ConfigParserCaseSens()
|
|
if data:
|
|
try:
|
|
cp.read_string(unicode(data))
|
|
for pkg_file in cp['timestamp']:
|
|
if not self.check_package_timestamp(
|
|
path.join(base_dn, pkg_file),
|
|
cp['timestamp'][pkg_file]):
|
|
status = self.BinhostStatus.Updating
|
|
data = ""
|
|
break
|
|
except (CPError, KeyError):
|
|
status = self.BinhostStatus.BadEnv
|
|
data = ""
|
|
break
|
|
break
|
|
except urllib2.URLError as e:
|
|
status = self.BinhostStatus.EnvNotFound
|
|
pass
|
|
except BaseException as e:
|
|
if isinstance(e, KeyboardInterrupt):
|
|
raise
|
|
status = self.BinhostStatus.UnknownError
|
|
data = ""
|
|
self.binhosts_data[binhost] = status, data
|
|
return self.binhosts_data[binhost]
|
|
|
|
re_revison = re.compile("\w+=(\w+)")
|
|
|
|
def get_timestamp(self, binhost):
|
|
"""
|
|
Возвращает таймстам полученный от сервера
|
|
"""
|
|
DAY = 60 * 60 * 24
|
|
OUTDATES = 10 * DAY
|
|
timestamp_file = path.join(binhost, self.ts_path)
|
|
try:
|
|
t = time.time()
|
|
data = _urlopen(timestamp_file,
|
|
timeout=self.timeout).read().strip()
|
|
if data.isdigit() and self.last_ts.isdigit():
|
|
return (data, int((time.time() - t) * 1000),
|
|
t - int(data) < OUTDATES,
|
|
int(data) < int(self.last_ts))
|
|
except urllib2.URLError as e:
|
|
pass
|
|
except BaseException as e:
|
|
if isinstance(e, KeyboardInterrupt):
|
|
raise
|
|
pass
|
|
return "0", -1, False, True
|
|
|
|
def generate_by_timestamp(self):
|
|
for host in self.binhost_list:
|
|
if host:
|
|
ts_content, duration, good, downgrade = \
|
|
self.get_timestamp(host)
|
|
if ts_content:
|
|
yield host, ts_content, str(duration), good, downgrade
|
|
|
|
def get_sorted(self):
|
|
if self.data is None:
|
|
self.data = []
|
|
for host, ts, t, good, downgrade in sorted(
|
|
self.generate_by_timestamp(),
|
|
# критерий сортировки между серверами
|
|
# актуальные сортируются по скорости
|
|
# неактульные по timestamp
|
|
# актуальность - 5 дней
|
|
reverse=True,
|
|
key=lambda x: ( # приоритетны актуальные (не более 5 дней)
|
|
x[3],
|
|
# приоритет для неактуальных
|
|
# самое свежее
|
|
0 if x[3] else int(x[1]),
|
|
# приоритет для неактуальных
|
|
# самое быстрое
|
|
0 if x[3] else -int(x[2]),
|
|
# приоритет для актуальных
|
|
# самое быстрое
|
|
-int(x[2]) if x[3] else 0,
|
|
# самое свежее
|
|
int(x[1]) if x[3] else 0)):
|
|
# rev_data = self.check_binhost(host)
|
|
self.data.append([host, ts, t, good, downgrade])
|
|
return self.data
|
|
|
|
def is_cache(self):
|
|
return self.data is not None
|
|
|
|
@classmethod
|
|
def param_id(cls, *args, **kw):
|
|
if not kw:
|
|
return ",".join(str(x) for x in args)
|
|
else:
|
|
return "%s|%s" % (",".join(str(x) for x in args),
|
|
",".join("%s:%s" % (str(k), str(v)) for k, v in
|
|
kw.items()))
|
|
|
|
class BinhostError(Exception):
|
|
pass
|
|
|
|
class Binhosts(BinhostsBase):
|
|
__metaclass__ = SingletonParam
|
|
|
|
class PackagesIndex(MutableMapping):
|
|
def __init__(self, data):
|
|
header, self.body = data.partition("\n\n")[::2]
|
|
self.header_dict = {}
|
|
for line in header.split('\n'):
|
|
k, v = line.partition(":")[::2]
|
|
self.header_dict[k] = v.strip()
|
|
|
|
def __getitem__(self, item):
|
|
return self.header_dict[item]
|
|
|
|
def __iter__(self):
|
|
return iter(self.header_dict)
|
|
|
|
def __len__(self):
|
|
return len(self.header_dict)
|
|
|
|
def __setitem__(self, key, value):
|
|
self.header_dict[key] = value
|
|
|
|
def __delitem__(self, key):
|
|
self.header_dict.pop(key)
|
|
|
|
def write(self, f):
|
|
f.write("\n".join("%s: %s" % (x, self.header_dict[x])
|
|
for x in sorted(self.header_dict.keys())))
|
|
f.write("\n\n%s"%self.body)
|
|
|
|
def fetch_packages(url_binhost, cache_fn, timeout=300, ttl=30 * DAYS):
|
|
"""
|
|
Получить файл Packages из бинарного хоста (распаковать если архив,
|
|
добавить поля DOWNLOAD_TIMESTAMP и TTL
|
|
:param url_binhost:
|
|
:param cache_fn:
|
|
:param timeout:
|
|
:param ttl:
|
|
:return:
|
|
"""
|
|
if path.exists(cache_fn):
|
|
try:
|
|
os.unlink(cache_fn)
|
|
except OSError:
|
|
pass
|
|
data = None
|
|
for uri in ("Packages.xz", "Packages"):
|
|
fn = path.join(url_binhost, uri)
|
|
try:
|
|
data = _urlopen(fn, timeout=timeout).read()
|
|
if uri == "Packages.xz":
|
|
data = xz(data, decompress=True)
|
|
break
|
|
except urllib2.URLError as e:
|
|
pass
|
|
except BaseException as e:
|
|
if isinstance(e, KeyboardInterrupt):
|
|
raise
|
|
data = ""
|
|
if not data:
|
|
raise BinhostError(_("Failed to fetch Packages from binary host %s")
|
|
% url_binhost)
|
|
pi = PackagesIndex(data)
|
|
pi["TTL"] = str(ttl)
|
|
pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time()))
|
|
try:
|
|
with writeFile(cache_fn) as f:
|
|
pi.write(f)
|
|
except (OSError, IOError):
|
|
raise BinhostError(_("Failed to save Packages"))
|
|
return True
|