calculate
/
calculate-utils-3-lib
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Переписан поиск ближайшего зеркала обновлений

Browse Source
legacy27 3.6.0_beta4
Mike Hiretsky 6 years ago
parent ded6530b81
commit 75a8a79137
6 changed files with 257 additions and 167 deletions
Show Stats Download Patch File Download Diff File

6
pym/calculate/lib/datavars.py
Unescape View File

@ -175,6 +175,7 @@ class Variable(VariableInterface):
Choice = "choice"
Table = "table"
Boolean = "bool"
Object = "object"
# label of variable
label = None
@ -219,6 +220,10 @@ class Variable(VariableInterface):
On = "on"
Off = "off"
EmptyTable = [[]]
Readonly = READONLY
Writable = WRITEABLE
def __init__(self, parent=None, **kwargs):
"""
@ -361,7 +366,6 @@ class Variable(VariableInterface):
return val
def _human(self):
# TODO: may be rest to NONE
oldModeGet = self.modeGet
try:
self.modeGet = Variable.HUMAN

364
pym/calculate/lib/utils/binhosts.py
Unescape View File

@ -21,9 +21,12 @@ import time
import os
from os import path
from calculate.lib.configparser import ConfigParserCaseSens, Error as CPError
from calculate.lib.utils.tools import SingletonParam
from calculate.lib.utils.tools import SingletonParam, Cachable
from calculate.lib.utils.gpg import GPGError
from .files import writeFile, xz
from collections import MutableMapping
from collections import MutableMapping, OrderedDict
from functools import total_ordering
import io
_ = lambda x: x
from calculate.lib.cl_lang import setLocalTranslate
@ -46,7 +49,7 @@ def _urlopen(fn, timeout=None):
return opener.open(fn)
class BinhostsBase(object):
class BinhostsBase(Cachable):
class BinhostStatus:
Success = 0
EnvNotFound = 1
@ -55,128 +58,178 @@ class BinhostsBase(object):
UnknownError = 4
def __init__(self, timeout, revision_path, ts_path, last_ts, binhost_list,
arch=None, skip_timestamp=False):
arch, gpg=None):
super(BinhostsBase,self).__init__()
self.timeout = int(timeout)
self.revision_path = revision_path
self.ts_path = ts_path
self.last_ts = last_ts
if last_ts.isdigit():
self.last_ts = int(last_ts)
else:
self.last_ts = 0
self.binhost_list = binhost_list
self.data = None
self.arch = arch
self.skip_timestamp = skip_timestamp
self.binhosts_data = {}
self.gpg = gpg
self.actual_period = 10 * DAYS
@Cachable.methodcached()
def fetch_package_timestamp(self, fn):
try:
for i, line in enumerate(
_urlopen(fn, timeout=self.timeout)):
if line.startswith("TIMESTAMP"):
return line.rpartition(":")[2].strip()
if i > 50:
break
except urllib2.URLError as e:
return ""
return ""
def check_package_timestamp(self, fn, timestamp):
for i, line in enumerate(
_urlopen(fn, timeout=self.timeout)):
if line.startswith("TIMESTAMP"):
return str(timestamp) == line.rpartition(":")[2].strip()
if i > 50:
break
return False
return str(timestamp) == self.fetch_package_timestamp(fn)
def check_binhost(self, binhost):
status = self.BinhostStatus.Success
if binhost not in self.binhosts_data:
revision_files = [path.join(binhost, x)
for x in self.revision_path]
data = ""
@Cachable.methodcached()
def fetch_envdata(self, binhost):
revision_files = (path.join(binhost, x)
for x in self.revision_path)
for fn in revision_files:
try:
for fn in revision_files:
base_dn = path.dirname(fn)
data = _urlopen(fn, timeout=self.timeout).read()
cp = ConfigParserCaseSens()
if data:
try:
cp.read_string(unicode(data))
for pkg_file in cp['timestamp']:
if not self.check_package_timestamp(
path.join(base_dn, pkg_file),
cp['timestamp'][pkg_file]):
status = self.BinhostStatus.Updating
data = ""
break
except (CPError, KeyError):
status = self.BinhostStatus.BadEnv
data = ""
break
break
return fn, _urlopen(fn, timeout=self.timeout).read()
except urllib2.URLError as e:
status = self.BinhostStatus.EnvNotFound
pass
return None, ""
return None, ""
def binhost_status(self, binhost):
fn, data = self.fetch_envdata(binhost)
if fn:
cp = ConfigParserCaseSens()
try:
cp.read_string(unicode(data))
base_dn = path.dirname(fn)
for pkg_file in cp['timestamp']:
if not self.check_package_timestamp(
path.join(base_dn, pkg_file),
cp['timestamp'][pkg_file]):
return self.BinhostStatus.Updating
except (CPError, KeyError):
return self.BinhostStatus.BadEnv
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
status = self.BinhostStatus.UnknownError
data = ""
self.binhosts_data[binhost] = status, data
return self.binhosts_data[binhost]
return self.BinhostStatus.UnknownError
return self.BinhostStatus.Success
else:
return self.BinhostStatus.EnvNotFound
@Cachable.methodcached()
def binhost_check_sign(self, binhost):
urlhost = "{}/grp/{}".format(binhost, self.arch)
try:
packages = Binhosts.fetch_packages(urlhost)
except BinhostError:
return False
try:
if self.gpg:
Binhosts.check_packages_signature(urlhost, packages, self.gpg)
return True
except BinhostSignError:
return False
re_revison = re.compile("\w+=(\w+)")
def get_timestamp(self, binhost):
def _get_timestamp(self, timestamp_file):
"""
Возвращает таймстам полученный от сервера
Получить timestamp с сервера обновлений
"""
DAY = 60 * 60 * 24
OUTDATES = 10 * DAY
timestamp_file = path.join(binhost, self.ts_path)
try:
t = time.time()
data = _urlopen(timestamp_file,
timeout=self.timeout).read().strip()
if data.isdigit() and self.last_ts.isdigit():
return (data, int((time.time() - t) * 1000),
t - int(data) < OUTDATES,
int(data) < int(self.last_ts))
except urllib2.URLError as e:
pass
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
pass
return "0", -1, False, True
def generate_by_timestamp(self):
for host in self.binhost_list:
if host:
ts_content, duration, good, downgrade = \
self.get_timestamp(host)
if ts_content:
yield host, ts_content, str(duration), good, downgrade
def get_sorted(self):
if self.data is None:
self.data = []
for host, ts, t, good, downgrade in sorted(
self.generate_by_timestamp(),
# критерий сортировки между серверами
# актуальные сортируются по скорости
# неактульные по timestamp
# актуальность - 5 дней
reverse=True,
key=lambda x: ( # приоритетны актуальные (не более 5 дней)
x[3],
# приоритет для неактуальных
# самое свежее
0 if x[3] else int(x[1]),
# приоритет для неактуальных
# самое быстрое
0 if x[3] else -int(x[2]),
# приоритет для актуальных
# самое быстрое
-int(x[2]) if x[3] else 0,
# самое свежее
int(x[1]) if x[3] else 0)):
# rev_data = self.check_binhost(host)
self.data.append([host, ts, t, good, downgrade])
return self.data
data = _urlopen(timestamp_file,
timeout=self.timeout).read().strip()
if not data.isdigit():
raise ValueError()
return int(data)
@total_ordering
class Binhost(object):
def __init__(self, parent, host):
start_ts = time.time()
self.host = host
self.parent = parent
try:
timestamp_file = path.join(self.host, self.parent.ts_path)
self.timestamp = self.parent._get_timestamp(timestamp_file)
self.duration = int((time.time() - start_ts) * 1000)
self.outdated = int(start_ts) - self.timestamp > parent.actual_period
self.downgraded = self.timestamp < parent.last_ts
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
self.timestamp = 0
self.duration = 0
self.outdated = True
self.downgraded = True
@property
def status(self):
return self.parent.binhost_status(self.host)
@property
def data(self):
return self.parent.fetch_envdata(self.host)[1]
@property
def valid(self):
return self.timestamp != 0
@property
def bad_sign(self):
return not self.parent.binhost_check_sign(self.host)
def __eq__(self, other):
if not self.valid and self.valid == other.valid:
return True
if self.valid != other.valid:
return False
return (self.outdated == other.outdated and self.duration == other.duration
and self.timestamp == self.timestamp)
def __lt__(self, other):
if self.valid:
if not other.valid:
return False
if self.outdated == other.outdated:
if self.outdated:
return (self.timestamp,-self.duration) < (other.timestamp,-other.duration)
else:
return (-self.duration,self.timestamp) < (-other.duration,other.timestamp)
return other.outdated
else:
return other.valid
def get_binhost(self, binhost):
"""
Получить от сервера время создания обновлений,
время затраченное на скачивание этого файла,
устарели или нет обновления
время создания обновление < текущих
"""
return self.Binhost(self, binhost)
@Cachable.methodcached()
def get_binhosts(self):
return [self.get_binhost(x) for x in self.binhost_list if x]
def is_cache(self):
return False
raise NotImplementedError("Need to revision")
return self.data is not None
@classmethod
def param_id(cls, *args, **kw):
"""
Метод для метакласса SingletonParam
"""
if not kw:
return ",".join(str(x) for x in args)
else:
@ -184,16 +237,72 @@ class BinhostsBase(object):
",".join("%s:%s" % (str(k), str(v)) for k, v in
kw.items()))
@staticmethod
def fetch_packages(url_binhost, timeout=300):
"""
Получить файл Packages из бинарного хоста (распаковать если архив,
добавить поля DOWNLOAD_TIMESTAMP и TTL
:param url_binhost:
:param cache_fn:
:param timeout:
:param ttl:
:return:
"""
data = None
data_asc = None
for uri in ("Packages.xz", "Packages"):
fn = path.join(url_binhost, uri)
try:
data = _urlopen(fn, timeout=timeout).read()
if uri == "Packages.xz":
data = xz(data, decompress=True)
break
except urllib2.URLError as e:
pass
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
data = ""
if not data:
raise BinhostError(_("Failed to fetch Packages from binary host %s")
% url_binhost)
return data
@staticmethod
def check_packages_signature(url_binhost, packages, gpg, timeout=300, sign=None):
"""
Проверить подпись индексного файла
"""
try:
sign = sign or Binhosts.fetch_packages_sign(url_binhost, timeout)
gpg.verify(packages, sign)
except GPGError as e:
raise BinhostSignError(_("Wrong Packages signature"))
@staticmethod
def fetch_packages_sign(url_binhost, timeout=300):
"""
Получить файл подписи Packages
"""
asc_fn = path.join(url_binhost, "Packages.asc")
try:
return _urlopen(asc_fn, timeout=timeout).read()
except urllib2.URLError as e:
raise BinhostSignError(_("Failed to fetch Packages signature"))
class BinhostError(Exception):
pass
class BinhostSignError(BinhostError):
pass
class Binhosts(BinhostsBase):
__metaclass__ = SingletonParam
class PackagesIndex(MutableMapping):
def __init__(self, data):
header, self.body = data.partition("\n\n")[::2]
self.header_dict = {}
self.header_dict = OrderedDict()
for line in header.split('\n'):
k, v = line.partition(":")[::2]
self.header_dict[k] = v.strip()
@ -213,49 +322,16 @@ class PackagesIndex(MutableMapping):
def __delitem__(self, key):
self.header_dict.pop(key)
def clean(self):
for k in ("TTL", "DOWNLOAD_TIMESTAMP"):
if k in self.header_dict:
self.header_dict.pop(k)
def get_value(self):
return "".join(("\n".join("%s: %s" % (x, self.header_dict[x])
for x in sorted(self.header_dict.keys())),
"\n\n%s"%self.body))
def write(self, f):
f.write("\n".join("%s: %s" % (x, self.header_dict[x])
for x in sorted(self.header_dict.keys())))
f.write("\n\n%s"%self.body)
f.write(self.get_value())
def fetch_packages(url_binhost, cache_fn, timeout=300, ttl=30 * DAYS):
"""
Получить файл Packages из бинарного хоста (распаковать если архив,
добавить поля DOWNLOAD_TIMESTAMP и TTL
:param url_binhost:
:param cache_fn:
:param timeout:
:param ttl:
:return:
"""
if path.exists(cache_fn):
try:
os.unlink(cache_fn)
except OSError:
pass
data = None
for uri in ("Packages.xz", "Packages"):
fn = path.join(url_binhost, uri)
try:
data = _urlopen(fn, timeout=timeout).read()
if uri == "Packages.xz":
data = xz(data, decompress=True)
break
except urllib2.URLError as e:
pass
except BaseException as e:
if isinstance(e, KeyboardInterrupt):
raise
data = ""
if not data:
raise BinhostError(_("Failed to fetch Packages from binary host %s")
% url_binhost)
pi = PackagesIndex(data)
pi["TTL"] = str(ttl)
pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time()))
try:
with writeFile(cache_fn) as f:
pi.write(f)
except (OSError, IOError):
raise BinhostError(_("Failed to save Packages"))
return True

7
pym/calculate/lib/utils/files.py
Unescape View File

@ -199,8 +199,11 @@ class process(StdoutableProcess):
def write(self, data):
"""Write to process stdin"""
self._open()
self.pipe.stdin.write(data)
self.pipe.stdin.flush()
try:
self.pipe.stdin.write(data)
self.pipe.stdin.flush()
except IOError as e:
raise FilesError(str(e))
def close(self):
"""Close stdin"""

2
pym/calculate/lib/utils/git.py
Unescape View File

@ -750,7 +750,7 @@ class Git(object):
git_dir = self._gitDir(rpath)
git_show = self.process(self._git, "--git-dir", git_dir, "log",
"-n1", "--format=format:%H",
"--quiet", reference, stderr=STDOUT)
"--quiet", reference)
if git_show.success():
return git_show.read().strip()
else:

37
pym/calculate/lib/utils/gpg.py
Unescape View File

@ -15,8 +15,13 @@
# limitations under the License.
from calculate.lib.utils.files import (process, getProgPath, makeDirectory,
removeDir)
removeDir, FilesError)
import os
import sys
_ = lambda x: x
from calculate.lib.cl_lang import setLocalTranslate
setLocalTranslate('cl_lib3', sys.modules[__name__])
class GPGError(Exception):
pass
@ -70,7 +75,7 @@ class GPG(object):
EXPIRED_PREFIX = "%s EXPKEYSIG" % GNUPREFIX
REVOCED_PREFIX = "%s REVKEYSIG" % GNUPREFIX
SIGDATA_PREFIX = "%s VALIDSIG" % GNUPREFIX
def __init__(self, homedir, timeout=20):
self.homedir = homedir
self.timeout = timeout
@ -84,6 +89,8 @@ class GPG(object):
return cmd
def _spawn_gpg(self, *options):
if not self.homedir:
raise GPGError(_("GPG is not initialized"))
return process(self.gpgcmd, "--status-fd", "1",
"--batch", "--no-autostart",
"--homedir", self.homedir, *options,
@ -94,18 +101,23 @@ class GPG(object):
def import_key(self, keyfile):
p = self._spawn_gpg("--import")
p.write(keyfile.read())
if not p.success():
try:
p.write(keyfile)
if not p.success():
raise GPGImportError(p.readerr())
except FilesError as e:
raise GPGImportError(p.readerr())
def verify(self, data, sign):
with Pipe() as datapipe, Pipe() as signpipe:
with Pipe() as signpipe:
p = self._spawn_gpg("--verify",
signpipe.get_filename(), datapipe.get_filename())
signpipe.write(sign.read())
signpipe.closein()
datapipe.write(data.read())
datapipe.closein()
signpipe.get_filename(), "-")
try:
signpipe.write(sign)
signpipe.closein()
p.write(data)
except FilesError as e:
raise GPGError(p.readerr())
goodsig = False
sig_data_ok = False
@ -125,8 +137,9 @@ class GPG(object):
else:
raise GPGError(p.readerr())
def keys_list(self):
pass
def count_public(self):
p = self._spawn_gpg("--list-public-keys")
return len([True for l in p if l.startswith("pub")])
def __enter__(self):
return self

8
pym/calculate/lib/utils/portage.py
Unescape View File

@ -1324,13 +1324,7 @@ class EbuildInfo(object):
return all(other[k] == self[k] for k in self.support_keys)
def __ne__(self, other):
res = any(other[k] != self[k] for k in self.support_keys)
# TODO: DEBUG
# if res:
# for k in self.support_keys:
# if other[k] != self[k]:
# print "DEBUG:", k,":", other[k], "!=", self[k]
return res
return any(other[k] != self[k] for k in self.support_keys)
class InstalledPackageInfoError(Exception):

Write Preview
Loading…
Cancel
Save