From 75a8a7913767a23922641867d8b60cb044f758ec Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Wed, 22 Aug 2018 16:23:15 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BF=D0=B8=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=20=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=20=D0=B1=D0=BB?= =?UTF-8?q?=D0=B8=D0=B6=D0=B0=D0=B9=D1=88=D0=B5=D0=B3=D0=BE=20=D0=B7=D0=B5?= =?UTF-8?q?=D1=80=D0=BA=D0=B0=D0=BB=D0=B0=20=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/calculate/lib/datavars.py | 6 +- pym/calculate/lib/utils/binhosts.py | 364 +++++++++++++++++----------- pym/calculate/lib/utils/files.py | 7 +- pym/calculate/lib/utils/git.py | 2 +- pym/calculate/lib/utils/gpg.py | 37 ++- pym/calculate/lib/utils/portage.py | 8 +- 6 files changed, 257 insertions(+), 167 deletions(-) diff --git a/pym/calculate/lib/datavars.py b/pym/calculate/lib/datavars.py index 7712a3e..de398b1 100644 --- a/pym/calculate/lib/datavars.py +++ b/pym/calculate/lib/datavars.py @@ -175,6 +175,7 @@ class Variable(VariableInterface): Choice = "choice" Table = "table" Boolean = "bool" + Object = "object" # label of variable label = None @@ -219,6 +220,10 @@ class Variable(VariableInterface): On = "on" Off = "off" + EmptyTable = [[]] + + Readonly = READONLY + Writable = WRITEABLE def __init__(self, parent=None, **kwargs): """ @@ -361,7 +366,6 @@ class Variable(VariableInterface): return val def _human(self): - # TODO: may be rest to NONE oldModeGet = self.modeGet try: self.modeGet = Variable.HUMAN diff --git a/pym/calculate/lib/utils/binhosts.py b/pym/calculate/lib/utils/binhosts.py index b1cf9af..a638d6a 100644 --- a/pym/calculate/lib/utils/binhosts.py +++ b/pym/calculate/lib/utils/binhosts.py @@ -21,9 +21,12 @@ import time import os from os import path from calculate.lib.configparser import ConfigParserCaseSens, Error as CPError -from calculate.lib.utils.tools import SingletonParam +from calculate.lib.utils.tools import SingletonParam, Cachable +from calculate.lib.utils.gpg import GPGError from .files import writeFile, xz -from collections import MutableMapping +from collections import MutableMapping, OrderedDict +from functools import total_ordering +import io _ = lambda x: x from calculate.lib.cl_lang import setLocalTranslate @@ -46,7 +49,7 @@ def _urlopen(fn, timeout=None): return opener.open(fn) -class BinhostsBase(object): +class BinhostsBase(Cachable): class BinhostStatus: Success = 0 EnvNotFound = 1 @@ -55,128 +58,178 @@ class BinhostsBase(object): UnknownError = 4 def __init__(self, timeout, revision_path, ts_path, last_ts, binhost_list, - arch=None, skip_timestamp=False): + arch, gpg=None): + super(BinhostsBase,self).__init__() self.timeout = int(timeout) self.revision_path = revision_path self.ts_path = ts_path - self.last_ts = last_ts + if last_ts.isdigit(): + self.last_ts = int(last_ts) + else: + self.last_ts = 0 self.binhost_list = binhost_list self.data = None self.arch = arch - self.skip_timestamp = skip_timestamp self.binhosts_data = {} + self.gpg = gpg + self.actual_period = 10 * DAYS + + @Cachable.methodcached() + def fetch_package_timestamp(self, fn): + try: + for i, line in enumerate( + _urlopen(fn, timeout=self.timeout)): + if line.startswith("TIMESTAMP"): + return line.rpartition(":")[2].strip() + if i > 50: + break + except urllib2.URLError as e: + return "" + return "" def check_package_timestamp(self, fn, timestamp): - for i, line in enumerate( - _urlopen(fn, timeout=self.timeout)): - if line.startswith("TIMESTAMP"): - return str(timestamp) == line.rpartition(":")[2].strip() - if i > 50: - break - return False + return str(timestamp) == self.fetch_package_timestamp(fn) - def check_binhost(self, binhost): - status = self.BinhostStatus.Success - if binhost not in self.binhosts_data: - revision_files = [path.join(binhost, x) - for x in self.revision_path] - data = "" + @Cachable.methodcached() + def fetch_envdata(self, binhost): + revision_files = (path.join(binhost, x) + for x in self.revision_path) + for fn in revision_files: try: - for fn in revision_files: - base_dn = path.dirname(fn) - data = _urlopen(fn, timeout=self.timeout).read() - cp = ConfigParserCaseSens() - if data: - try: - cp.read_string(unicode(data)) - for pkg_file in cp['timestamp']: - if not self.check_package_timestamp( - path.join(base_dn, pkg_file), - cp['timestamp'][pkg_file]): - status = self.BinhostStatus.Updating - data = "" - break - except (CPError, KeyError): - status = self.BinhostStatus.BadEnv - data = "" - break - break + return fn, _urlopen(fn, timeout=self.timeout).read() except urllib2.URLError as e: - status = self.BinhostStatus.EnvNotFound - pass + return None, "" + return None, "" + + def binhost_status(self, binhost): + fn, data = self.fetch_envdata(binhost) + if fn: + cp = ConfigParserCaseSens() + try: + cp.read_string(unicode(data)) + base_dn = path.dirname(fn) + for pkg_file in cp['timestamp']: + if not self.check_package_timestamp( + path.join(base_dn, pkg_file), + cp['timestamp'][pkg_file]): + return self.BinhostStatus.Updating + except (CPError, KeyError): + return self.BinhostStatus.BadEnv except BaseException as e: if isinstance(e, KeyboardInterrupt): raise - status = self.BinhostStatus.UnknownError - data = "" - self.binhosts_data[binhost] = status, data - return self.binhosts_data[binhost] + return self.BinhostStatus.UnknownError + return self.BinhostStatus.Success + else: + return self.BinhostStatus.EnvNotFound + + @Cachable.methodcached() + def binhost_check_sign(self, binhost): + urlhost = "{}/grp/{}".format(binhost, self.arch) + try: + packages = Binhosts.fetch_packages(urlhost) + except BinhostError: + return False + try: + if self.gpg: + Binhosts.check_packages_signature(urlhost, packages, self.gpg) + return True + except BinhostSignError: + return False re_revison = re.compile("\w+=(\w+)") - def get_timestamp(self, binhost): + def _get_timestamp(self, timestamp_file): """ - Возвращает таймстам полученный от сервера + Получить timestamp с сервера обновлений """ - DAY = 60 * 60 * 24 - OUTDATES = 10 * DAY - timestamp_file = path.join(binhost, self.ts_path) - try: - t = time.time() - data = _urlopen(timestamp_file, - timeout=self.timeout).read().strip() - if data.isdigit() and self.last_ts.isdigit(): - return (data, int((time.time() - t) * 1000), - t - int(data) < OUTDATES, - int(data) < int(self.last_ts)) - except urllib2.URLError as e: - pass - except BaseException as e: - if isinstance(e, KeyboardInterrupt): - raise - pass - return "0", -1, False, True - - def generate_by_timestamp(self): - for host in self.binhost_list: - if host: - ts_content, duration, good, downgrade = \ - self.get_timestamp(host) - if ts_content: - yield host, ts_content, str(duration), good, downgrade - - def get_sorted(self): - if self.data is None: - self.data = [] - for host, ts, t, good, downgrade in sorted( - self.generate_by_timestamp(), - # критерий сортировки между серверами - # актуальные сортируются по скорости - # неактульные по timestamp - # актуальность - 5 дней - reverse=True, - key=lambda x: ( # приоритетны актуальные (не более 5 дней) - x[3], - # приоритет для неактуальных - # самое свежее - 0 if x[3] else int(x[1]), - # приоритет для неактуальных - # самое быстрое - 0 if x[3] else -int(x[2]), - # приоритет для актуальных - # самое быстрое - -int(x[2]) if x[3] else 0, - # самое свежее - int(x[1]) if x[3] else 0)): - # rev_data = self.check_binhost(host) - self.data.append([host, ts, t, good, downgrade]) - return self.data + data = _urlopen(timestamp_file, + timeout=self.timeout).read().strip() + if not data.isdigit(): + raise ValueError() + return int(data) + + @total_ordering + class Binhost(object): + def __init__(self, parent, host): + start_ts = time.time() + self.host = host + self.parent = parent + try: + timestamp_file = path.join(self.host, self.parent.ts_path) + self.timestamp = self.parent._get_timestamp(timestamp_file) + self.duration = int((time.time() - start_ts) * 1000) + self.outdated = int(start_ts) - self.timestamp > parent.actual_period + self.downgraded = self.timestamp < parent.last_ts + except BaseException as e: + if isinstance(e, KeyboardInterrupt): + raise + self.timestamp = 0 + self.duration = 0 + self.outdated = True + self.downgraded = True + + @property + def status(self): + return self.parent.binhost_status(self.host) + + @property + def data(self): + return self.parent.fetch_envdata(self.host)[1] + + @property + def valid(self): + return self.timestamp != 0 + + @property + def bad_sign(self): + return not self.parent.binhost_check_sign(self.host) + + def __eq__(self, other): + if not self.valid and self.valid == other.valid: + return True + if self.valid != other.valid: + return False + return (self.outdated == other.outdated and self.duration == other.duration + and self.timestamp == self.timestamp) + + def __lt__(self, other): + if self.valid: + if not other.valid: + return False + if self.outdated == other.outdated: + if self.outdated: + return (self.timestamp,-self.duration) < (other.timestamp,-other.duration) + else: + return (-self.duration,self.timestamp) < (-other.duration,other.timestamp) + return other.outdated + else: + return other.valid + + + def get_binhost(self, binhost): + """ + Получить от сервера время создания обновлений, + время затраченное на скачивание этого файла, + устарели или нет обновления + время создания обновление < текущих + """ + return self.Binhost(self, binhost) + + @Cachable.methodcached() + def get_binhosts(self): + return [self.get_binhost(x) for x in self.binhost_list if x] def is_cache(self): + return False + raise NotImplementedError("Need to revision") return self.data is not None @classmethod def param_id(cls, *args, **kw): + """ + Метод для метакласса SingletonParam + """ if not kw: return ",".join(str(x) for x in args) else: @@ -184,16 +237,72 @@ class BinhostsBase(object): ",".join("%s:%s" % (str(k), str(v)) for k, v in kw.items())) + @staticmethod + def fetch_packages(url_binhost, timeout=300): + """ + Получить файл Packages из бинарного хоста (распаковать если архив, + добавить поля DOWNLOAD_TIMESTAMP и TTL + :param url_binhost: + :param cache_fn: + :param timeout: + :param ttl: + :return: + """ + data = None + data_asc = None + for uri in ("Packages.xz", "Packages"): + fn = path.join(url_binhost, uri) + try: + data = _urlopen(fn, timeout=timeout).read() + if uri == "Packages.xz": + data = xz(data, decompress=True) + break + except urllib2.URLError as e: + pass + except BaseException as e: + if isinstance(e, KeyboardInterrupt): + raise + data = "" + if not data: + raise BinhostError(_("Failed to fetch Packages from binary host %s") + % url_binhost) + return data + + @staticmethod + def check_packages_signature(url_binhost, packages, gpg, timeout=300, sign=None): + """ + Проверить подпись индексного файла + """ + try: + sign = sign or Binhosts.fetch_packages_sign(url_binhost, timeout) + gpg.verify(packages, sign) + except GPGError as e: + raise BinhostSignError(_("Wrong Packages signature")) + + @staticmethod + def fetch_packages_sign(url_binhost, timeout=300): + """ + Получить файл подписи Packages + """ + asc_fn = path.join(url_binhost, "Packages.asc") + try: + return _urlopen(asc_fn, timeout=timeout).read() + except urllib2.URLError as e: + raise BinhostSignError(_("Failed to fetch Packages signature")) + class BinhostError(Exception): pass +class BinhostSignError(BinhostError): + pass + class Binhosts(BinhostsBase): __metaclass__ = SingletonParam class PackagesIndex(MutableMapping): def __init__(self, data): header, self.body = data.partition("\n\n")[::2] - self.header_dict = {} + self.header_dict = OrderedDict() for line in header.split('\n'): k, v = line.partition(":")[::2] self.header_dict[k] = v.strip() @@ -213,49 +322,16 @@ class PackagesIndex(MutableMapping): def __delitem__(self, key): self.header_dict.pop(key) + def clean(self): + for k in ("TTL", "DOWNLOAD_TIMESTAMP"): + if k in self.header_dict: + self.header_dict.pop(k) + + def get_value(self): + return "".join(("\n".join("%s: %s" % (x, self.header_dict[x]) + for x in sorted(self.header_dict.keys())), + "\n\n%s"%self.body)) + def write(self, f): - f.write("\n".join("%s: %s" % (x, self.header_dict[x]) - for x in sorted(self.header_dict.keys()))) - f.write("\n\n%s"%self.body) + f.write(self.get_value()) -def fetch_packages(url_binhost, cache_fn, timeout=300, ttl=30 * DAYS): - """ - Получить файл Packages из бинарного хоста (распаковать если архив, - добавить поля DOWNLOAD_TIMESTAMP и TTL - :param url_binhost: - :param cache_fn: - :param timeout: - :param ttl: - :return: - """ - if path.exists(cache_fn): - try: - os.unlink(cache_fn) - except OSError: - pass - data = None - for uri in ("Packages.xz", "Packages"): - fn = path.join(url_binhost, uri) - try: - data = _urlopen(fn, timeout=timeout).read() - if uri == "Packages.xz": - data = xz(data, decompress=True) - break - except urllib2.URLError as e: - pass - except BaseException as e: - if isinstance(e, KeyboardInterrupt): - raise - data = "" - if not data: - raise BinhostError(_("Failed to fetch Packages from binary host %s") - % url_binhost) - pi = PackagesIndex(data) - pi["TTL"] = str(ttl) - pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time())) - try: - with writeFile(cache_fn) as f: - pi.write(f) - except (OSError, IOError): - raise BinhostError(_("Failed to save Packages")) - return True diff --git a/pym/calculate/lib/utils/files.py b/pym/calculate/lib/utils/files.py index 8fc5d68..95f8297 100644 --- a/pym/calculate/lib/utils/files.py +++ b/pym/calculate/lib/utils/files.py @@ -199,8 +199,11 @@ class process(StdoutableProcess): def write(self, data): """Write to process stdin""" self._open() - self.pipe.stdin.write(data) - self.pipe.stdin.flush() + try: + self.pipe.stdin.write(data) + self.pipe.stdin.flush() + except IOError as e: + raise FilesError(str(e)) def close(self): """Close stdin""" diff --git a/pym/calculate/lib/utils/git.py b/pym/calculate/lib/utils/git.py index 5f3b063..7dc52ff 100644 --- a/pym/calculate/lib/utils/git.py +++ b/pym/calculate/lib/utils/git.py @@ -750,7 +750,7 @@ class Git(object): git_dir = self._gitDir(rpath) git_show = self.process(self._git, "--git-dir", git_dir, "log", "-n1", "--format=format:%H", - "--quiet", reference, stderr=STDOUT) + "--quiet", reference) if git_show.success(): return git_show.read().strip() else: diff --git a/pym/calculate/lib/utils/gpg.py b/pym/calculate/lib/utils/gpg.py index 487d143..c9e5b66 100644 --- a/pym/calculate/lib/utils/gpg.py +++ b/pym/calculate/lib/utils/gpg.py @@ -15,8 +15,13 @@ # limitations under the License. from calculate.lib.utils.files import (process, getProgPath, makeDirectory, - removeDir) + removeDir, FilesError) import os +import sys +_ = lambda x: x +from calculate.lib.cl_lang import setLocalTranslate + +setLocalTranslate('cl_lib3', sys.modules[__name__]) class GPGError(Exception): pass @@ -70,7 +75,7 @@ class GPG(object): EXPIRED_PREFIX = "%s EXPKEYSIG" % GNUPREFIX REVOCED_PREFIX = "%s REVKEYSIG" % GNUPREFIX SIGDATA_PREFIX = "%s VALIDSIG" % GNUPREFIX - + def __init__(self, homedir, timeout=20): self.homedir = homedir self.timeout = timeout @@ -84,6 +89,8 @@ class GPG(object): return cmd def _spawn_gpg(self, *options): + if not self.homedir: + raise GPGError(_("GPG is not initialized")) return process(self.gpgcmd, "--status-fd", "1", "--batch", "--no-autostart", "--homedir", self.homedir, *options, @@ -94,18 +101,23 @@ class GPG(object): def import_key(self, keyfile): p = self._spawn_gpg("--import") - p.write(keyfile.read()) - if not p.success(): + try: + p.write(keyfile) + if not p.success(): + raise GPGImportError(p.readerr()) + except FilesError as e: raise GPGImportError(p.readerr()) def verify(self, data, sign): - with Pipe() as datapipe, Pipe() as signpipe: + with Pipe() as signpipe: p = self._spawn_gpg("--verify", - signpipe.get_filename(), datapipe.get_filename()) - signpipe.write(sign.read()) - signpipe.closein() - datapipe.write(data.read()) - datapipe.closein() + signpipe.get_filename(), "-") + try: + signpipe.write(sign) + signpipe.closein() + p.write(data) + except FilesError as e: + raise GPGError(p.readerr()) goodsig = False sig_data_ok = False @@ -125,8 +137,9 @@ class GPG(object): else: raise GPGError(p.readerr()) - def keys_list(self): - pass + def count_public(self): + p = self._spawn_gpg("--list-public-keys") + return len([True for l in p if l.startswith("pub")]) def __enter__(self): return self diff --git a/pym/calculate/lib/utils/portage.py b/pym/calculate/lib/utils/portage.py index e2a0c9f..3b41b05 100644 --- a/pym/calculate/lib/utils/portage.py +++ b/pym/calculate/lib/utils/portage.py @@ -1324,13 +1324,7 @@ class EbuildInfo(object): return all(other[k] == self[k] for k in self.support_keys) def __ne__(self, other): - res = any(other[k] != self[k] for k in self.support_keys) - # TODO: DEBUG - # if res: - # for k in self.support_keys: - # if other[k] != self[k]: - # print "DEBUG:", k,":", other[k], "!=", self[k] - return res + return any(other[k] != self[k] for k in self.support_keys) class InstalledPackageInfoError(Exception):