|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# Copyright 2015-2016 Mir Calculate. http://www.calculate-linux.org
|
|
|
#
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
# You may obtain a copy of the License at
|
|
|
#
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
#
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
# See the License for the specific language governing permissions and
|
|
|
# limitations under the License.
|
|
|
|
|
|
import sys
|
|
|
import urllib.request as urllib2
|
|
|
import re
|
|
|
import time
|
|
|
import os
|
|
|
from os import path
|
|
|
from ..configparser import ConfigParserCaseSens, Error as CPError
|
|
|
from .tools import SingletonParam, Cachable
|
|
|
from .gpg import GPGError
|
|
|
from .files import writeFile, xz
|
|
|
from collections import OrderedDict
|
|
|
from collections.abc import MutableMapping
|
|
|
from functools import total_ordering
|
|
|
from contextlib import contextmanager
|
|
|
import io
|
|
|
import threading
|
|
|
|
|
|
_ = lambda x: x
|
|
|
from ..cl_lang import setLocalTranslate
|
|
|
|
|
|
setLocalTranslate('cl_lib3', sys.modules[__name__])
|
|
|
|
|
|
MINUTES = 60
|
|
|
HOURS = 60 * MINUTES
|
|
|
DAYS = 24 * HOURS
|
|
|
|
|
|
@contextmanager
|
|
|
def _urlopen(fn, timeout=None):
|
|
|
"""
|
|
|
Получить URL, без прокси
|
|
|
"""
|
|
|
proxy_handler = urllib2.ProxyHandler({})
|
|
|
opener = urllib2.build_opener(proxy_handler)
|
|
|
opener.addheaders = [('User-Agent', 'Calculate Linux')]
|
|
|
if timeout is not None:
|
|
|
f = opener.open(fn, timeout=timeout)
|
|
|
else:
|
|
|
f = opener.open(fn)
|
|
|
yield f
|
|
|
|
|
|
t = threading.Thread(target=f.close)
|
|
|
t.daemon = True
|
|
|
t.start()
|
|
|
|
|
|
class BinhostsBase(Cachable):
|
|
|
class BinhostStatus:
|
|
|
Success = 0
|
|
|
EnvNotFound = 1
|
|
|
Updating = 2
|
|
|
BadEnv = 3
|
|
|
UnknownError = 4
|
|
|
|
|
|
def __init__(self, timeout, revision_path, ts_path, last_ts, binhost_list,
|
|
|
arch, gpg=None, base=False):
|
|
|
super().__init__()
|
|
|
self.timeout = int(timeout)
|
|
|
self.revision_path = revision_path
|
|
|
self.ts_path = ts_path
|
|
|
if last_ts.isdigit():
|
|
|
self.last_ts = int(last_ts)
|
|
|
else:
|
|
|
self.last_ts = 0
|
|
|
self.binhost_list = binhost_list
|
|
|
self.data = None
|
|
|
self.arch = arch
|
|
|
self.binhosts_data = {}
|
|
|
self.gpg = gpg
|
|
|
self.actual_period = 10 * DAYS
|
|
|
self.base = base
|
|
|
|
|
|
@Cachable.methodcached()
|
|
|
def fetch_package_timestamp(self, fn):
|
|
|
try:
|
|
|
with _urlopen(fn, timeout=self.timeout) as f:
|
|
|
for i, line in enumerate(f):
|
|
|
if line.startswith(b"TIMESTAMP"):
|
|
|
return line.rpartition(b":")[2].strip()
|
|
|
if i > 50:
|
|
|
break
|
|
|
except urllib2.URLError as e:
|
|
|
return b""
|
|
|
return b""
|
|
|
|
|
|
def check_package_timestamp(self, fn, timestamp):
|
|
|
return timestamp.encode("UTF-8") == self.fetch_package_timestamp(fn)
|
|
|
|
|
|
@Cachable.methodcached()
|
|
|
def fetch_envdata(self, binhost):
|
|
|
revision_files = (path.join(binhost, x)
|
|
|
for x in self.revision_path)
|
|
|
for fn in revision_files:
|
|
|
try:
|
|
|
with _urlopen(fn, timeout=self.timeout) as f:
|
|
|
return fn, f.read().decode("UTF-8")
|
|
|
except urllib2.URLError as e:
|
|
|
return None, ""
|
|
|
return None, ""
|
|
|
|
|
|
def binhost_status(self, binhost):
|
|
|
fn, data = self.fetch_envdata(binhost)
|
|
|
if fn:
|
|
|
cp = ConfigParserCaseSens()
|
|
|
try:
|
|
|
cp.read_string(data)
|
|
|
base_dn = path.dirname(fn)
|
|
|
for pkg_file in cp['timestamp']:
|
|
|
if not self.check_package_timestamp(
|
|
|
path.join(base_dn, pkg_file),
|
|
|
cp['timestamp'][pkg_file]):
|
|
|
return self.BinhostStatus.Updating
|
|
|
except (CPError, KeyError) as e:
|
|
|
return self.BinhostStatus.BadEnv
|
|
|
except BaseException as e:
|
|
|
if isinstance(e, KeyboardInterrupt):
|
|
|
raise
|
|
|
return self.BinhostStatus.UnknownError
|
|
|
return self.BinhostStatus.Success
|
|
|
else:
|
|
|
return self.BinhostStatus.EnvNotFound
|
|
|
|
|
|
@Cachable.methodcached()
|
|
|
def binhost_check_sign(self, binhost):
|
|
|
urlhost = "{}/grp/{}".format(binhost, self.arch)
|
|
|
try:
|
|
|
packages = Binhosts.fetch_packages(urlhost)
|
|
|
except BinhostError:
|
|
|
return False
|
|
|
try:
|
|
|
if self.gpg:
|
|
|
Binhosts.check_packages_signature(urlhost, packages, self.gpg)
|
|
|
return True
|
|
|
except BinhostSignError:
|
|
|
return False
|
|
|
|
|
|
re_revison = re.compile("\w+=(\w+)")
|
|
|
|
|
|
def _get_timestamp(self, timestamp_file):
|
|
|
"""
|
|
|
Получить timestamp с сервера обновлений
|
|
|
"""
|
|
|
with _urlopen(timestamp_file, timeout=self.timeout) as f:
|
|
|
data = f.read().strip()
|
|
|
if not data.isdigit():
|
|
|
raise ValueError()
|
|
|
return int(data)
|
|
|
|
|
|
@total_ordering
|
|
|
class Binhost():
|
|
|
def __init__(self, parent, host):
|
|
|
start_ts = time.time()
|
|
|
self.host = host
|
|
|
self.parent = parent
|
|
|
try:
|
|
|
timestamp_file = path.join(self.host, self.parent.ts_path)
|
|
|
self.timestamp = self.parent._get_timestamp(timestamp_file)
|
|
|
self.duration = int((time.time() - start_ts) * 1000)
|
|
|
self.outdated = int(start_ts) - self.timestamp > parent.actual_period
|
|
|
self.downgraded = self.timestamp < parent.last_ts
|
|
|
|
|
|
fn, data = self.parent.fetch_envdata(self.host)
|
|
|
if fn:
|
|
|
cp = ConfigParserCaseSens()
|
|
|
try:
|
|
|
cp.read_string(data)
|
|
|
if "update" in cp.sections():
|
|
|
self.level = cp["update"]["level"]
|
|
|
else:
|
|
|
self.level = 99999
|
|
|
except (CPError, KeyError) as e:
|
|
|
self.level = 99999
|
|
|
else:
|
|
|
self.level = 99999
|
|
|
except BaseException as e:
|
|
|
if isinstance(e, KeyboardInterrupt):
|
|
|
raise
|
|
|
self.timestamp = 0
|
|
|
self.duration = 0
|
|
|
self.outdated = True
|
|
|
self.downgraded = True
|
|
|
|
|
|
@property
|
|
|
def status(self):
|
|
|
return self.parent.binhost_status(self.host)
|
|
|
|
|
|
@property
|
|
|
def data(self):
|
|
|
return self.parent.fetch_envdata(self.host)[1]
|
|
|
|
|
|
@property
|
|
|
def valid(self):
|
|
|
return self.timestamp != 0
|
|
|
|
|
|
@property
|
|
|
def bad_sign(self):
|
|
|
return not self.parent.binhost_check_sign(self.host)
|
|
|
|
|
|
def __eq__(self, other):
|
|
|
if not self.valid and self.valid == other.valid:
|
|
|
return True
|
|
|
if self.valid != other.valid:
|
|
|
return False
|
|
|
return (self.outdated == other.outdated and self.duration == other.duration
|
|
|
and self.timestamp == self.timestamp)
|
|
|
|
|
|
def __lt__(self, other):
|
|
|
if self.valid:
|
|
|
if not other.valid:
|
|
|
return False
|
|
|
if self.outdated == other.outdated:
|
|
|
if self.outdated:
|
|
|
return (self.timestamp,-self.duration) < (other.timestamp,-other.duration)
|
|
|
else:
|
|
|
return (-self.duration,self.timestamp) < (-other.duration,other.timestamp)
|
|
|
return other.outdated
|
|
|
else:
|
|
|
return other.valid
|
|
|
|
|
|
|
|
|
class BaseBinhost(Binhost):
|
|
|
"""
|
|
|
Не проводить проверку базового бинхоста
|
|
|
"""
|
|
|
def __init__(self, parent, host):
|
|
|
super().__init__(parent, host)
|
|
|
self.outdated = False
|
|
|
self.downgraded = False
|
|
|
|
|
|
@property
|
|
|
def valid(self):
|
|
|
return True
|
|
|
|
|
|
@property
|
|
|
def status(self):
|
|
|
return BinhostsBase.BinhostStatus.Success
|
|
|
|
|
|
|
|
|
def get_binhost(self, binhost):
|
|
|
"""
|
|
|
Получить от сервера время создания обновлений,
|
|
|
время затраченное на скачивание этого файла,
|
|
|
устарели или нет обновления
|
|
|
время создания обновление < текущих
|
|
|
"""
|
|
|
if self.base:
|
|
|
return self.BaseBinhost(self, binhost)
|
|
|
else:
|
|
|
return self.Binhost(self, binhost)
|
|
|
|
|
|
@Cachable.methodcached()
|
|
|
def get_binhosts(self):
|
|
|
return [self.get_binhost(x) for x in self.binhost_list if x]
|
|
|
|
|
|
def is_cache(self):
|
|
|
return False
|
|
|
raise NotImplementedError("Need to revision")
|
|
|
return self.data is not None
|
|
|
|
|
|
@classmethod
|
|
|
def param_id(cls, *args, **kw):
|
|
|
"""
|
|
|
Метод для метакласса SingletonParam
|
|
|
"""
|
|
|
if not kw:
|
|
|
return ",".join(str(x) for x in args)
|
|
|
else:
|
|
|
return "%s|%s" % (",".join(str(x) for x in args),
|
|
|
",".join("%s:%s" % (str(k), str(v)) for k, v in
|
|
|
kw.items()))
|
|
|
|
|
|
@staticmethod
|
|
|
def fetch_packages(url_binhost, timeout=300):
|
|
|
"""
|
|
|
Получить файл Packages из бинарного хоста (распаковать если архив,
|
|
|
добавить поля DOWNLOAD_TIMESTAMP и TTL
|
|
|
:param url_binhost:
|
|
|
:param cache_fn:
|
|
|
:param timeout:
|
|
|
:param ttl:
|
|
|
:return:
|
|
|
"""
|
|
|
data = None
|
|
|
for uri in ("Packages.xz", "Packages"):
|
|
|
fn = path.join(url_binhost, uri)
|
|
|
try:
|
|
|
with _urlopen(fn, timeout=timeout) as f:
|
|
|
data = f.read()
|
|
|
if uri == "Packages.xz":
|
|
|
data = xz(data, decompress=True)
|
|
|
break
|
|
|
except urllib2.URLError as e:
|
|
|
pass
|
|
|
except BaseException as e:
|
|
|
if isinstance(e, KeyboardInterrupt):
|
|
|
raise
|
|
|
data = b""
|
|
|
if not data:
|
|
|
raise BinhostError(_("Failed to fetch Packages from binary host %s")
|
|
|
% url_binhost)
|
|
|
return data.decode("UTF-8")
|
|
|
|
|
|
@staticmethod
|
|
|
def check_packages_signature(url_binhost, packages, gpg, timeout=300, sign=None):
|
|
|
"""
|
|
|
Проверить подпись индексного файла
|
|
|
"""
|
|
|
try:
|
|
|
sign = sign or Binhosts.fetch_packages_sign(url_binhost, timeout)
|
|
|
gpg.verify(packages, sign)
|
|
|
except GPGError as e:
|
|
|
raise BinhostSignError(_("Wrong Packages signature"))
|
|
|
|
|
|
@staticmethod
|
|
|
def fetch_packages_sign(url_binhost, timeout=300):
|
|
|
"""
|
|
|
Получить файл подписи Packages
|
|
|
"""
|
|
|
asc_fn = path.join(url_binhost, "Packages.asc")
|
|
|
try:
|
|
|
with _urlopen(asc_fn, timeout=timeout) as f:
|
|
|
return f.read()
|
|
|
except urllib2.URLError as e:
|
|
|
raise BinhostSignError(_("Failed to fetch Packages signature"))
|
|
|
|
|
|
class BinhostError(Exception):
|
|
|
pass
|
|
|
|
|
|
class BinhostSignError(BinhostError):
|
|
|
pass
|
|
|
|
|
|
class Binhosts(BinhostsBase, metaclass=SingletonParam):
|
|
|
pass
|
|
|
|
|
|
class PackagesIndex(MutableMapping):
|
|
|
def __init__(self, data):
|
|
|
header, self.body = data.partition("\n\n")[::2]
|
|
|
self.header_dict = OrderedDict()
|
|
|
for line in header.split('\n'):
|
|
|
k, v = line.partition(":")[::2]
|
|
|
self.header_dict[k] = v.strip()
|
|
|
|
|
|
def __getitem__(self, item):
|
|
|
return self.header_dict[item]
|
|
|
|
|
|
def __iter__(self):
|
|
|
return iter(self.header_dict)
|
|
|
|
|
|
def __len__(self):
|
|
|
return len(self.header_dict)
|
|
|
|
|
|
def __setitem__(self, key, value):
|
|
|
self.header_dict[key] = value
|
|
|
|
|
|
def __delitem__(self, key):
|
|
|
self.header_dict.pop(key)
|
|
|
|
|
|
def clean(self):
|
|
|
for k in ("TTL", "DOWNLOAD_TIMESTAMP"):
|
|
|
if k in self.header_dict:
|
|
|
self.header_dict.pop(k)
|
|
|
|
|
|
def get_value(self):
|
|
|
return "".join(("\n".join("%s: %s" % (x, self.header_dict[x])
|
|
|
for x in sorted(self.header_dict.keys())),
|
|
|
"\n\n%s"%self.body))
|
|
|
|
|
|
def write(self, f):
|
|
|
f.write(self.get_value())
|
|
|
|
