You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

695 lines
22 KiB

  1. #-*- coding: utf-8 -*-
  2. # Copyright 2014 Calculate Ltd. http://www.calculate-linux.org
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. import hashlib
  16. import os
  17. from os import path
  18. import re
  19. import sys
  20. from calculate.lib.utils.colortext.palette import TextState
  21. from calculate.lib.utils.tools import ignore
  22. from package_tools import EmergePackage, PackageList, EmergeUpdateInfo, \
  23. EmergeRemoveInfo, Git, GitError
  24. Colors = TextState.Colors
  25. import pexpect
  26. from calculate.lib.utils.files import getProgPath, readLinesFile, listDirectory, \
  27. writeFile, readFile
  28. from calculate.lib.utils.colortext.output import XmlOutput
  29. from calculate.lib.utils.colortext.converter import (ConsoleCodes256Converter,
  30. XmlConverter)
  31. from calculate.lib.cl_log import log
  32. from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _
  33. setLocalTranslate('cl_update3', sys.modules[__name__])
  34. __ = getLazyLocalTranslate(_)
  35. class EmergeError(Exception):
  36. """
  37. Ошибка при сборке пакетов
  38. """
  39. class EmergeNeedRootError(EmergeError):
  40. pass
  41. class CommandExecutor(object):
  42. """
  43. Запуск программы для объекта Emerge
  44. """
  45. logfile = '/var/log/calculate/lastcommand.log'
  46. def __init__(self, cmd, params, env=None, cwd=None, logfile=None):
  47. self.cwd = cwd
  48. self.env = env or dict(os.environ)
  49. self.env.update({'EINFO_QUIET':'NO'})
  50. self.cmd = cmd
  51. self.params = params
  52. self.child = None
  53. if logfile:
  54. self.logfile = logfile
  55. def execute(self):
  56. if self.child is None:
  57. self.child = pexpect.spawn(self.cmd,
  58. logfile=open(self.logfile, 'w'),
  59. env=self.env, cwd=self.cwd, timeout=None)
  60. return self.child
  61. def close(self):
  62. if self.child is not None:
  63. self.child.close()
  64. self.child = None
  65. def success(self):
  66. if self.child:
  67. if self.child.isalive():
  68. self.child.wait()
  69. return self.child.exitstatus == 0
  70. return False
  71. def failed(self):
  72. return not self.success()
  73. def send(self, s):
  74. if self.child:
  75. self.child.send(s)
  76. class EmergeCommand(CommandExecutor):
  77. """
  78. Запуск emerge для последующего анализирования
  79. """
  80. # параметры по умолчанию
  81. default_params = ["-av", "--color=y", "--nospinner"]
  82. cmd = getProgPath("/usr/bin/emerge")
  83. def __init__(self, packages, extra_params=None, env=None, cwd=None,
  84. logfile=None, emerge_default_opts=None):
  85. extra_params = extra_params or []
  86. self.child = None
  87. self.packages = packages
  88. self.params = self.default_params + extra_params
  89. wrong_default_opts = ("--columns","--ask ", "--ask=")
  90. if emerge_default_opts is None:
  91. default_env = {'CLEAN_DELAY': '0'}
  92. else:
  93. default_env = {
  94. 'CLEAN_DELAY': '0',
  95. 'EMERGE_DEFAULT_OPTS': " ".join(filter(
  96. lambda x: not any(y in x for y in wrong_default_opts),
  97. emerge_default_opts))
  98. }
  99. default_env.update(os.environ)
  100. self.env = env or default_env
  101. self.cwd = cwd
  102. if logfile:
  103. self.logfile = logfile
  104. def execute(self):
  105. if self.child is None:
  106. self.child = pexpect.spawn(self.cmd, self.params + self.packages,
  107. logfile=open(self.logfile, 'w'),
  108. env=self.env, cwd=self.cwd, timeout=None)
  109. return self.child
  110. class EmergeInformationBlock(object):
  111. _color_block = "(?:\033\[[^m]+?m)?"
  112. _new_line = "\r*\n"
  113. token = None
  114. end_token = ["\n"]
  115. re_block = None
  116. action = None
  117. re_match_type = type(re.match("", ""))
  118. re_type = type(re.compile(""))
  119. def __init__(self, parent):
  120. """
  121. :type parent: EmergeParser
  122. """
  123. self.result = None
  124. self.text_converter = parent.text_converter
  125. self.parent = parent
  126. self.parent.add_element(self)
  127. self.children = []
  128. def add_element(self, element):
  129. self.children.append(element)
  130. def __str__(self):
  131. if type(self.result) == self.re_match_type:
  132. return self.result.group()
  133. else:
  134. return self.result or ""
  135. def __nonzero__(self):
  136. return bool(self.result)
  137. def __len__(self):
  138. if self.result is None:
  139. return 0
  140. else:
  141. return len(self.result)
  142. def __contains__(self, item):
  143. if self.result is None:
  144. return False
  145. else:
  146. return item in str(self)
  147. def _get_text(self, result):
  148. """
  149. Получить результат из регулярки и преобразовать его через self.converter
  150. """
  151. if result:
  152. return self.text_converter.transform(result.rstrip())
  153. return ""
  154. def get_block(self, child):
  155. try:
  156. token = child.match
  157. if type(self.end_token) == self.re_type:
  158. child.expect(self.end_token)
  159. match = child.match.group()
  160. else:
  161. child.expect_exact(self.end_token)
  162. match = child.match
  163. self.get_data(self.re_block.search(
  164. token + child.before + match))
  165. except pexpect.EOF:
  166. child.buffer = "".join(
  167. [x for x in (child.before, child.after, child.buffer)
  168. if type(x) == str])
  169. def get_data(self, match):
  170. self.result = self._get_text(match.group(1))
  171. class InstallPackagesBlock(EmergeInformationBlock):
  172. """
  173. Блок emerge содержащий список пакетов для установки
  174. """
  175. list = PackageList([])
  176. remove_list = PackageList([])
  177. _new_line = EmergeInformationBlock._new_line
  178. token = "\n["
  179. end_token = ["\r\n\r", "\n\n"]
  180. re_block = re.compile(r"((?:^\[.*?{nl})+)".format(nl=_new_line),
  181. re.MULTILINE)
  182. def get_data(self, match):
  183. super(InstallPackagesBlock, self).get_data(match)
  184. list_block = XmlConverter().transform(self.result).split('\n')
  185. self.list = PackageList(map(EmergeUpdateInfo, list_block))
  186. self.remove_list = PackageList(map(EmergeRemoveInfo, list_block))
  187. class UninstallPackagesBlock(EmergeInformationBlock):
  188. """
  189. Блок emerge содержащий список удаляемых пакетов
  190. """
  191. list = PackageList([])
  192. verbose_result = ""
  193. _new_line = EmergeInformationBlock._new_line
  194. _color_block = EmergeInformationBlock._color_block
  195. token = ["Calculating removal order",
  196. "These are the packages that would be unmerged",]
  197. end_token = re.compile("All selected packages:.*\n")
  198. re_block = re.compile(
  199. r"(?:{token}).*?{nl}(.*){nl}All selected packages: (.*?){nl}".
  200. format(token="|".join(token),
  201. nl=_new_line, c=_color_block), re.DOTALL)
  202. def get_data(self, match):
  203. re_clean = re.compile(
  204. "^.*?({token}).*?{c}{nl}".format(token="|".join(self.token),
  205. nl=self._new_line,
  206. c=self._color_block), re.DOTALL)
  207. verbose_result = re_clean.sub("", match.group(1))
  208. self.verbose_result = self._get_text(verbose_result)
  209. self.result = self._get_text(match.group(2))
  210. list_block = XmlConverter().transform(self.result).split()
  211. self.list = PackageList(map(EmergePackage, list_block))
  212. class FinishEmergeGroup(EmergeInformationBlock):
  213. """
  214. Блок завершения команды
  215. """
  216. token = pexpect.EOF
  217. # регуляреное выражение, определяющее содержит ли блок
  218. # сообщения об ошибках
  219. re_failed = re.compile(
  220. r"Fetch instructions for \S+:|"
  221. r"The following.*are necessary to proceed|"
  222. r"!!! Multiple package .* slot have been pulled|"
  223. r"no ebuilds to satisfy|"
  224. r"Dependencies could not be completely resolved due to",
  225. re.MULTILINE)
  226. def get_block(self, child):
  227. if child.isalive():
  228. child.wait()
  229. if child.exitstatus != 0 or self.re_failed.search(child.before):
  230. self.children_get_block(child)
  231. else:
  232. self.result = True
  233. def children_get_block(self, child):
  234. for block in self.children:
  235. block.get_block(child)
  236. def children_action(self, child):
  237. for block in (x for x in self.children if x.result and x.action):
  238. if block.action(child) is False:
  239. break
  240. def action(self, child):
  241. self.children_action(child)
  242. return False
  243. class PrepareErrorBlock(EmergeInformationBlock):
  244. """
  245. Блок информации с ошибками при получении списка устанавливаемых пакетов
  246. """
  247. token = None
  248. re_drop = re.compile("news items need reading|"
  249. "Use eselect news|"
  250. "Calculating dependencies|"
  251. "to read news items|"
  252. "Local copy of remote index is up-to-date|"
  253. "These are the packages that would be merged|"
  254. "Process finished with exit code")
  255. re_multi_empty_line = re.compile("(?:<br/>){3,}", re.DOTALL)
  256. re_strip_br = re.compile("^(?:<br/>)+|(?:<br/>)+$", re.DOTALL)
  257. def remove_needless_data(self, data):
  258. return "\n".join([x for x in data.split('\n')
  259. if not self.re_drop.search(x)])
  260. def strip_br(self, data):
  261. return self.re_strip_br.sub(
  262. "",
  263. self.re_multi_empty_line.sub("<br/><br/>", data))
  264. def get_block(self, child):
  265. self.result = self.strip_br(
  266. self._get_text(self.remove_needless_data(child.before)))
  267. def action(self, child):
  268. raise EmergeError(_("Emerge failed"))
  269. class DownloadSizeBlock(EmergeInformationBlock):
  270. """
  271. Размер скачиваемых обновлений
  272. """
  273. token = "Size of downloads:"
  274. re_block = re.compile(r"Size of downloads:\s(\S+\s\S+)")
  275. def __str__(self):
  276. if self.result:
  277. return self.result
  278. else:
  279. return "0 kB"
  280. class QuestionBlock(EmergeInformationBlock):
  281. """
  282. Блок вопроса
  283. """
  284. default_answer = "yes"
  285. _color_block = EmergeInformationBlock._color_block
  286. token = "Would you like"
  287. end_token = ["]", "\n"]
  288. re_block = re.compile(
  289. "(Would you.*)\[{c}Yes{c}/{c}No{c}".format(c=_color_block))
  290. def action(self, child):
  291. if self.result:
  292. child.send("%s\n" % self.default_answer)
  293. class NeedRootBlock(EmergeInformationBlock):
  294. """
  295. Пользователь не явеляется root
  296. """
  297. token = "This action requires superuser access"
  298. def get_data(self, child):
  299. self.result = True
  300. def action(self, child):
  301. raise EmergeNeedRootError(_("This action requires superuser access"))
  302. class NotifierInformationBlock(EmergeInformationBlock):
  303. """
  304. Информационный блок поддерживающий observing
  305. """
  306. def __init__(self, parent):
  307. super(NotifierInformationBlock, self).__init__(parent)
  308. self.observers = []
  309. def get_data(self, match):
  310. self.result = match
  311. def add_observer(self, f):
  312. self.observers.append(f)
  313. def notify(self, observer, groups):
  314. observer(groups)
  315. def action(self, child):
  316. if self.result and self.observers:
  317. groups = self.result.groups()
  318. for observer in self.observers:
  319. self.notify(observer, groups)
  320. class EmergingPackage(NotifierInformationBlock):
  321. """
  322. Запуск устанавливаемого пакета
  323. ObserverFunc: (package, num=number, max_num=number, binary=binary)
  324. """
  325. _color_block = EmergeInformationBlock._color_block
  326. token = ">>> Emerging "
  327. re_block = re.compile(
  328. "Emerging (binary )?\({c}(\d+){c} "
  329. "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
  330. def notify(self, observer, groups):
  331. observer(EmergePackage(groups[3]), num=groups[1], max_num=groups[2],
  332. binary=bool(groups[0]))
  333. class UnemergingPackage(NotifierInformationBlock):
  334. """
  335. Запуск устанавливаемого пакета
  336. ObserverFunc: (package, num=number, max_num=number)
  337. """
  338. _color_block = EmergeInformationBlock._color_block
  339. token = ">>> Unmerging"
  340. re_block = re.compile(
  341. r"Unmerging (?:\({c}(\d+){c} "
  342. r"of {c}(\d+){c}\) )?(\S+)\.\.\.".format(c=_color_block))
  343. def notify(self, observer, groups):
  344. observer(EmergePackage(groups[2]), num=groups[0], max_num=groups[1])
  345. class FetchingTarball(NotifierInformationBlock):
  346. """
  347. Происходит скачивание архивов
  348. """
  349. token = "Saving to:"
  350. re_block = re.compile("Saving to:\s*‘(\S+)?’")
  351. def notify(self, observer, groups):
  352. observer(groups[0])
  353. class InstallingPackage(NotifierInformationBlock):
  354. """
  355. Запуск устанавливаемого пакета
  356. ObserverFunc: (package, binary=binary)
  357. """
  358. _color_block = EmergeInformationBlock._color_block
  359. binary = None
  360. token = ">>> Installing "
  361. re_block = re.compile(
  362. "Installing \({c}(\d+){c} "
  363. "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
  364. def notify(self, observer, groups):
  365. binary = bool(self.binary and groups[2] in self.binary)
  366. observer(EmergePackage(groups[2]), binary=binary)
  367. def mark_binary(self, package):
  368. if self.binary is None:
  369. self.binary = []
  370. self.binary.append(str(package))
  371. class EmergeingErrorBlock(EmergeInformationBlock):
  372. """
  373. Блок содержит информацию об ошибке во время сборки пакета
  374. """
  375. token = ["* ERROR: ", " * \033[39;49;00mERROR: "]
  376. end_token = "Working directory:"
  377. re_block = re.compile("ERROR: (\S*) failed \([^)]+\).*?"
  378. "The complete build log is located at '([^']+)",
  379. re.DOTALL)
  380. package = ""
  381. def get_data(self, match):
  382. self.result = self._get_text(match.group(2).rstrip())
  383. self.package = match.group(1)
  384. @property
  385. def log(self):
  386. return self.text_converter.transform(readFile(self.result))
  387. def action(self, child):
  388. raise EmergeError(_("Failed to emerge %s") % self.package)
  389. class RevdepPercentBlock(NotifierInformationBlock):
  390. """
  391. Блок определния статуса revdep-rebuild
  392. """
  393. token = "Collecting system binaries"
  394. end_token = [re.compile("Assigning files to packages|"
  395. "All prepared. Starting rebuild")]
  396. re_block = re.compile("\[\s(\d+)%\s\]")
  397. action = None
  398. def notify(self, observer, groups):
  399. percent = int(groups[0])
  400. observer(percent)
  401. def get_block(self, child):
  402. expect_result = [self.re_block]+self.end_token
  403. try:
  404. while True:
  405. index = child.expect(expect_result)
  406. if index == 0:
  407. for observer in self.observers:
  408. self.notify(observer, child.match.groups())
  409. else:
  410. self.result = child.match
  411. break
  412. except pexpect.EOF:
  413. self.result = ""
  414. class EmergeParser(object):
  415. """
  416. Парсер вывода emerge
  417. """
  418. def __init__(self, command, run=False):
  419. self.text_converter = ConsoleCodes256Converter(XmlOutput())
  420. self.command = command
  421. self.elements = {}
  422. self.install_packages = InstallPackagesBlock(self)
  423. self.uninstall_packages = UninstallPackagesBlock(self)
  424. self.question = QuestionBlock(self)
  425. self.finish_block = FinishEmergeGroup(self)
  426. self.need_root = NeedRootBlock(self)
  427. self.prepare_error = PrepareErrorBlock(self.finish_block)
  428. self.download_size = DownloadSizeBlock(self)
  429. self.emerging_error = EmergeingErrorBlock(self)
  430. self.installing = InstallingPackage(self)
  431. self.uninstalling = UnemergingPackage(self)
  432. self.emerging = EmergingPackage(self)
  433. self.fetching = FetchingTarball(self)
  434. self.emerging.add_observer(self.mark_binary)
  435. self.emerging.add_observer(self.skip_fetching)
  436. if run:
  437. self.run()
  438. def mark_binary(self, package, binary=False, **kw):
  439. if binary:
  440. self.installing.mark_binary(package)
  441. def skip_fetching(self, *argv, **kw):
  442. self.fetching.action = lambda child: None
  443. def add_element(self, element):
  444. if element.token:
  445. if type(element.token) == list:
  446. for token in element.token:
  447. self.elements[token] = element
  448. else:
  449. self.elements[element.token] = element
  450. def run(self):
  451. """
  452. Запустить команду
  453. """
  454. child = self.command.execute()
  455. while True:
  456. index = child.expect_exact(self.elements.keys())
  457. element = self.elements.values()[index]
  458. element.get_block(child)
  459. if element.action:
  460. if element.action(child) is False:
  461. break
  462. def close(self):
  463. self.command.close()
  464. def __enter__(self):
  465. return self
  466. def __exit__(self, *exc_info):
  467. self.close()
  468. class MtimeCheckvalue(object):
  469. def __init__(self, *fname):
  470. self.fname = fname
  471. def value_func(self, fn):
  472. return str(int(os.stat(fn).st_mtime))
  473. def get_check_values(self, file_list):
  474. for fn in file_list:
  475. if path.exists(fn) and not path.isdir(fn):
  476. yield fn, self.value_func(fn)
  477. else:
  478. for k, v in self.get_check_values(
  479. listDirectory(fn, fullPath=True)):
  480. yield k, v
  481. def checkvalues(self):
  482. return self.get_check_values(self.fname)
  483. class Md5Checkvalue(MtimeCheckvalue):
  484. def value_func(self, fn):
  485. return hashlib.md5(readFile(fn)).hexdigest()
  486. class GitCheckvalue(object):
  487. def __init__(self, rpath):
  488. self.rpath = rpath
  489. self.git = Git()
  490. def checkvalues(self):
  491. with ignore(GitError):
  492. if self.git.is_git(self.rpath):
  493. yield self.rpath, Git().getCurrentCommit(self.rpath)
  494. class EmergeCache(object):
  495. """
  496. Кэш пакетов
  497. """
  498. cache_file = '/var/lib/calculate/calculate-update/world.cache'
  499. # список файлов проверяемый по mtime на изменения
  500. check_list = [MtimeCheckvalue('/etc/make.conf',
  501. '/etc/portage',
  502. '/etc/make.profile'),
  503. Md5Checkvalue('/var/lib/portage/world',
  504. '/var/lib/portage/world_sets')]
  505. logger = log("emerge-cache",
  506. filename="/var/log/calculate/emerge-cache.log",
  507. formatter="%(asctime)s - %(levelname)s - %(message)s")
  508. def __init__(self):
  509. self.files_control_values = {}
  510. self.pkg_list = PackageList([])
  511. def set_cache(self, package_list):
  512. """
  513. Установить в кэш список пакетов
  514. """
  515. with writeFile(self.cache_file) as f:
  516. for fn, val in self.get_control_values().items():
  517. f.write("{fn}={val}\n".format(fn=fn, val=val))
  518. f.write('\n')
  519. for pkg in package_list:
  520. f.write("%s\n"% str(pkg))
  521. self.logger.info("Setting cache (%d packages)"%len(package_list))
  522. def drop_cache(self, reason=None):
  523. if path.exists(self.cache_file):
  524. with ignore(OSError):
  525. os.unlink(self.cache_file)
  526. self.logger.info("Droping cache. Reason: %s"%reason)
  527. else:
  528. self.logger.info("Droping empty cache. Reason: %s"%reason)
  529. def get_cached_package_list(self):
  530. self.read_cache()
  531. if not path.exists(self.cache_file):
  532. self.logger.info("Requesting empty cache")
  533. if self.check_actuality():
  534. return self.pkg_list
  535. return None
  536. def check_actuality(self):
  537. """
  538. Кэш считается актуальным если ни один из файлов не менялся
  539. """
  540. if self.get_control_values() == self.files_control_values:
  541. self.logger.info(
  542. "Getting actuality cache (%d packages)" % len(self.pkg_list))
  543. return True
  544. else:
  545. reason = "Unknown"
  546. for k,v in self.get_control_values().items():
  547. if k in self.files_control_values:
  548. if v != self.files_control_values[k]:
  549. reason = "%s was modified"%k
  550. else:
  551. reason = "Checksum of file %s is not exist" % k
  552. self.logger.info("Failed to get cache. Reason: %s" % reason)
  553. return False
  554. def read_cache(self):
  555. self.files_control_values = {}
  556. cache_file_lines = readLinesFile(self.cache_file)
  557. for line in cache_file_lines:
  558. if not "=" in line:
  559. break
  560. k, v = line.split('=')
  561. self.files_control_values[k] = v.strip()
  562. self.pkg_list = PackageList(cache_file_lines)
  563. def get_control_values(self):
  564. def generate():
  565. for obj in self.check_list:
  566. for checkvalue in obj.checkvalues():
  567. yield checkvalue
  568. return dict(generate())