You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1262 lines
51 KiB

  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 Mir Calculate. http://www.calculate-linux.org
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from functools import wraps
  16. import random
  17. import sys
  18. from os import path
  19. import os
  20. import time
  21. from calculate.core.server.gen_pid import search_worked_process
  22. from calculate.core.setup_cache import Cache as SetupCache
  23. from calculate.core.server.func import MethodsInterface
  24. from calculate.lib.cl_template import SystemIni, LayeredIni
  25. from calculate.lib.datavars import DataVarsError
  26. from calculate.lib.utils.tools import AddonError
  27. from calculate.lib.utils.colortext.palette import TextState
  28. from calculate.lib.utils.colortext import get_color_print
  29. from calculate.update.emerge_parser import RevdepPercentBlock
  30. from calculate.update.datavars import DataVarsUpdate
  31. from calculate.update.update_info import UpdateInfo
  32. from calculate.lib.cl_log import log
  33. import re
  34. import shutil
  35. from collections import MutableSet
  36. from update_tasks import EmergeMark
  37. from calculate.lib.utils.git import Git, GitError, MTimeKeeper, NotGitError
  38. from calculate.lib.utils.portage import (Layman, EmergeLog,
  39. EmergeLogNamedTask, PackageList,
  40. PackageInformation,
  41. get_packages_files_directory,
  42. get_manifest_files_directory,
  43. get_remove_list)
  44. Colors = TextState.Colors
  45. from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
  46. PercentProgress, process, getRunCommands,
  47. readFile, listDirectory, pathJoin,
  48. writeFile)
  49. import emerge_parser
  50. import logging
  51. from emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
  52. EmergeCache, Chroot)
  53. from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
  54. RegexpLocalization, _)
  55. setLocalTranslate('cl_update3', sys.modules[__name__])
  56. __ = getLazyLocalTranslate(_)
  57. class UpdateError(AddonError):
  58. """Update Error"""
  59. class OverlayOwnCache(MutableSet):
  60. """
  61. Сет оверлеев с интегрированным кэшем
  62. """
  63. def __init__(self, dv=None):
  64. self.dv = dv
  65. def __get_overlays(self):
  66. own_cache_value = SystemIni(self.dv).getVar('update', 'own_cache') or ""
  67. return [x.strip() for x in own_cache_value.split(',') if x.strip()]
  68. def __write_overlays(self, overlays):
  69. if not overlays:
  70. SystemIni(self.dv).delVar('update', 'own_cache')
  71. else:
  72. SystemIni(self.dv).setVar('update',
  73. {'own_cache': ",".join(overlays)})
  74. def __contains__(self, item):
  75. return item in self.__get_overlays()
  76. def __iter__(self):
  77. return iter(self.__get_overlays())
  78. def __len__(self):
  79. return len(self.__get_overlays())
  80. def __append_value(self, overlays, value):
  81. if value not in overlays:
  82. overlays.append(value)
  83. self.__write_overlays(overlays)
  84. def add(self, value):
  85. overlays = self.__get_overlays()
  86. self.__append_value(overlays, value)
  87. def discard(self, value):
  88. overlays = self.__get_overlays()
  89. if value in overlays:
  90. overlays.remove(value)
  91. self.__write_overlays(overlays)
  92. def variable_module(var_env):
  93. def variable_module_decor(f):
  94. @wraps(f)
  95. def wrapper(self, *args, **kw):
  96. old_env = self.clVars.defaultModule
  97. try:
  98. self.clVars.defaultModule = var_env
  99. return f(self, *args, **kw)
  100. finally:
  101. self.clVars.defaultModule = old_env
  102. return wrapper
  103. return variable_module_decor
  104. class Update(MethodsInterface):
  105. """Основной объект для выполнения действий связанных с обновлением системы
  106. """
  107. def init(self):
  108. commandLog = path.join(self.clVars.Get('core.cl_log_path'),
  109. 'lastcommand.log')
  110. emerge_parser.CommandExecutor.logfile = commandLog
  111. self.color_print = get_color_print()
  112. self.emerge_cache = EmergeCache()
  113. if self.clVars.Get('cl_env_debug_set') == 'off':
  114. EmergeCache.logger.logger.setLevel(logging.WARNING)
  115. self.emerge_cache.check_list = (
  116. self.emerge_cache.check_list +
  117. map(emerge_parser.GitCheckvalue,
  118. self.clVars.Get('update.cl_update_rep_path')))
  119. self.update_map = {}
  120. self.refresh_binhost = False
  121. def get_prog_path(self, program_name):
  122. return getProgPath(program_name)
  123. def _syncRepository(self, name, url, rpath, revision,
  124. cb_progress=None, clean=False):
  125. """
  126. Синхронизировать репозитори
  127. """
  128. dv = self.clVars
  129. git = Git()
  130. info_outdated = False
  131. old_dir = "%s.old" % git._gitDir(rpath)
  132. if path.exists(old_dir):
  133. clean = True
  134. try:
  135. self.stash_cache(rpath, name)
  136. if not git.checkExistsRep(rpath):
  137. git.cloneTagRepository(url, rpath, revision,
  138. cb_progress=cb_progress)
  139. info_outdated = True
  140. else:
  141. cr = ""
  142. try:
  143. need_update = False
  144. tag_cr = git.getCommit(rpath, revision)
  145. cr = git.getCurrentCommit(rpath)
  146. ref_type = git.reference_type(rpath, revision)
  147. if git.isNeedUnpack(rpath):
  148. need_update = True
  149. clean = True
  150. elif tag_cr != cr or ref_type == Git.Reference.Branch:
  151. need_update = True
  152. elif clean:
  153. status = git.getStatusInfo(rpath)
  154. if not status or status['files']:
  155. need_update = True
  156. except GitError:
  157. need_update = True
  158. if need_update:
  159. git.updateTagRepository(url, rpath, revision,
  160. cb_progress=cb_progress,
  161. clean=clean)
  162. new_cr = git.getCurrentCommit(rpath)
  163. if new_cr != cr:
  164. info_outdated = True
  165. if info_outdated:
  166. self.raiseOutdate()
  167. dv.Set('cl_update_outdate_set', 'on', force=True)
  168. finally:
  169. self.unstash_cache(rpath, name)
  170. return True
  171. def raiseOutdate(self):
  172. self.clVars.Set('cl_update_outdate_set', 'on', force=True)
  173. def setAutocheckParams(self, status, interval, update_other, cleanpkg):
  174. """
  175. Настроить параметры автопроверки обновлений
  176. """
  177. onoff = lambda x: "on" if x else "off"
  178. self.clVars.Write('cl_update_autocheck_set', onoff(status), True)
  179. self.clVars.Write('cl_update_autocheck_interval', interval, True)
  180. self.clVars.Write('cl_update_other_set', onoff(update_other), True)
  181. self.clVars.Write('cl_update_cleanpkg_set', onoff(cleanpkg), True)
  182. if not status:
  183. UpdateInfo.set_update_ready(False)
  184. return True
  185. def checkSchedule(self, interval, status):
  186. """
  187. Проверить по расписанию необходимость запуска команды
  188. """
  189. if not status:
  190. self.printWARNING(_("Updates autocheck is not enabled"))
  191. return False
  192. last_check = SystemIni(self.clVars).getVar('update', 'last_check') or ""
  193. re_interval = re.compile("^(\d+)\s*(hours?|days?|weeks?)?", re.I)
  194. interval_match = re_interval.search(interval)
  195. MINUTE = 60
  196. HOUR = MINUTE * 60
  197. DAY = HOUR * 24
  198. WEEK = DAY * 7
  199. if interval_match:
  200. if interval_match.group(2):
  201. suffix_map = {'h': HOUR, 'd': DAY, 'w': WEEK}
  202. k = suffix_map.get(interval_match.group(2).lower()[0], HOUR)
  203. else:
  204. k = HOUR
  205. est = int(interval_match.group(1)) * k
  206. else:
  207. est = 3 * HOUR
  208. if last_check:
  209. if last_check.isdigit():
  210. if (time.time() - int(last_check)) < (est - 10 * MINUTE):
  211. self.printWARNING(_("Please wait for the update time"))
  212. return False
  213. self.mark_schedule()
  214. return True
  215. def checkRun(self, wait_update):
  216. """
  217. Проверить повторный запуск
  218. """
  219. update_running = lambda: any(os.getpid() != x
  220. for x in
  221. search_worked_process('update', dv))
  222. dv = self.clVars
  223. if update_running():
  224. if not wait_update:
  225. raise UpdateError(_("Update is already running. "
  226. "Try to run later."))
  227. else:
  228. self.startTask(_("Waiting for another update to be complete"))
  229. while update_running():
  230. self.pauseProcess()
  231. while update_running():
  232. time.sleep(0.3)
  233. self.resumeProcess()
  234. time.sleep(random.random() * 3)
  235. self.endTask()
  236. if self.clVars.Get('cl_chroot_status') == 'off':
  237. emerge_running = lambda: any("/usr/bin/emerge" in x
  238. for x in getRunCommands(True))
  239. if emerge_running():
  240. if not wait_update:
  241. raise UpdateError(_("Emerge is running. "
  242. "Try to run later."))
  243. else:
  244. self.startTask(_("Waiting for emerge to be complete"))
  245. while emerge_running():
  246. time.sleep(1)
  247. self.endTask()
  248. return True
  249. @variable_module("update")
  250. def trimRepositories(self, repname):
  251. """
  252. Синхронизировать репозитории
  253. """
  254. dv = self.clVars
  255. rpath = \
  256. dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1)
  257. git = Git()
  258. self.addProgress()
  259. git.trimRepository(rpath, cb_progress=self.setProgress)
  260. return True
  261. @variable_module("update")
  262. def syncRepositories(self, repname, clean_on_error=True):
  263. """
  264. Синхронизировать репозитории
  265. """
  266. dv = self.clVars
  267. check_status = dv.GetBool('update.cl_update_check_rep_set')
  268. url, rpath, revision = (
  269. dv.Select(["cl_update_rep_url", "cl_update_rep_path",
  270. "cl_update_rep_rev"],
  271. where="cl_update_rep_name", eq=repname, limit=1))
  272. if not url or not rpath:
  273. raise UpdateError(_("Configuration variables for repositories "
  274. "are not setup"))
  275. git = Git()
  276. if not git.checkUrl(url):
  277. raise UpdateError(_("Git %s is unavailable") % url)
  278. chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
  279. if chroot_path == '/':
  280. rpath_orig = rpath
  281. else:
  282. rpath_orig = rpath[len(chroot_path):]
  283. self.addProgress()
  284. mtime = MTimeKeeper(path.join(rpath, "profiles/updates"))
  285. mtime.save()
  286. try:
  287. if clean_on_error:
  288. try:
  289. layman = Layman(dv.Get('cl_update_layman_installed'),
  290. dv.Get('cl_update_layman_make'),
  291. dv.Get('cl_update_layman_conf'),
  292. prefix=chroot_path)
  293. if repname not in ("portage", "gentoo"):
  294. layman.add(repname, url, rpath_orig)
  295. if not self._syncRepository(repname, url, rpath, revision,
  296. cb_progress=self.setProgress,
  297. clean=check_status):
  298. return "skip"
  299. return True
  300. except GitError as e:
  301. if not isinstance(e, NotGitError):
  302. if e.addon:
  303. self.printWARNING(str(e.addon))
  304. self.printWARNING(str(e))
  305. self.endTask(False)
  306. self.startTask(
  307. _("Re-fetching the {name} repository").format(
  308. name=repname))
  309. self.addProgress()
  310. rpath_new = "%s_new" % rpath
  311. try:
  312. self._syncRepository(repname, url, rpath_new, revision,
  313. cb_progress=self.setProgress,
  314. clean=check_status)
  315. removeDir(rpath)
  316. shutil.move(rpath_new, rpath)
  317. except OSError as e:
  318. raise UpdateError(_("Failed to modify the "
  319. "{repname} repository").format(
  320. repname=repname) + _(": ") + str(e))
  321. finally:
  322. if path.exists(rpath_new):
  323. removeDir(rpath_new)
  324. else:
  325. if not self._syncRepository(repname, url, rpath, revision,
  326. clean=check_status):
  327. return "skip"
  328. layman = Layman(dv.Get('cl_update_layman_installed'),
  329. dv.Get('cl_update_layman_make'),
  330. dv.Get('cl_update_layman_conf'),
  331. prefix=chroot_path)
  332. if repname not in ("portage", "gentoo"):
  333. layman.add(repname, url, rpath_orig)
  334. finally:
  335. mtime.restore()
  336. return True
  337. metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
  338. def stash_cache(self, rpath, name):
  339. """
  340. Спрятать кэш
  341. """
  342. if name in ("portage", "gentoo"):
  343. return
  344. if not name in OverlayOwnCache(self.clVars):
  345. for cachename in self.metadata_cache_names:
  346. cachedir = path.join(rpath, cachename)
  347. if path.exists(cachedir):
  348. try:
  349. cachedir_s = path.join(path.dirname(rpath),
  350. path.basename(
  351. cachename) + ".stash")
  352. if path.exists(cachedir_s):
  353. removeDir(cachedir_s)
  354. shutil.move(cachedir, cachedir_s)
  355. except BaseException as e:
  356. pass
  357. def unstash_cache(self, rpath, name):
  358. """
  359. Извлеч кэш
  360. """
  361. if name in ("portage", "gentoo"):
  362. return
  363. cachenames = self.metadata_cache_names
  364. if not name in OverlayOwnCache(self.clVars):
  365. if any(path.exists(path.join(rpath, x)) for x in cachenames):
  366. for cachename in cachenames:
  367. cachedir_s = path.join(path.dirname(rpath),
  368. path.basename(cachename) + ".stash")
  369. if path.exists(cachedir_s):
  370. try:
  371. removeDir(cachedir_s)
  372. except BaseException as e:
  373. pass
  374. OverlayOwnCache(self.clVars).add(name)
  375. else:
  376. for cachename in cachenames:
  377. cachedir = path.join(rpath, cachename)
  378. cachedir_s = path.join(path.dirname(rpath),
  379. path.basename(cachename) + ".stash")
  380. if path.exists(cachedir_s):
  381. try:
  382. shutil.move(cachedir_s, cachedir)
  383. except BaseException as e:
  384. pass
  385. else:
  386. if all(not path.exists(path.join(rpath, x)) for x in cachenames):
  387. OverlayOwnCache(self.clVars).discard(name)
  388. def syncLaymanRepository(self, repname):
  389. """
  390. Обновить репозиторий через layman
  391. """
  392. layman = self.get_prog_path('/usr/bin/layman')
  393. if not layman:
  394. raise UpdateError(_("The Layman tool is not found"))
  395. rpath = self.clVars.Select('cl_update_other_rep_path',
  396. where='cl_update_other_rep_name', eq=repname,
  397. limit=1)
  398. laymanname = path.basename(rpath)
  399. self.stash_cache(rpath, laymanname)
  400. try:
  401. if Git.is_git(rpath):
  402. self.addProgress()
  403. p = PercentProgress(layman, "-s", laymanname, part=1, atty=True)
  404. for perc in p.progress():
  405. self.setProgress(perc)
  406. else:
  407. p = process(layman, "-s", repname, stderr=STDOUT)
  408. if p.failed():
  409. raise UpdateError(
  410. _("Failed to update the {rname} repository").format(
  411. rname=repname),
  412. addon=p.read())
  413. finally:
  414. self.unstash_cache(rpath, laymanname)
  415. return True
  416. def _regenCache_process(self, progname, repname, cpu_num):
  417. return process(progname, "--repo=%s" % repname, "--update",
  418. "--jobs=%s" % cpu_num, stderr=STDOUT)
  419. def regenCache(self, repname):
  420. """
  421. Обновить кэш метаданных репозитория
  422. """
  423. egenCache = self.get_prog_path('/usr/bin/egencache')
  424. if not egenCache:
  425. raise UpdateError(_("The Portage tool is not found"))
  426. if repname in self.clVars.Get('cl_update_rep_name'):
  427. path_rep = self.clVars.Select('cl_update_rep_path',
  428. where='cl_update_rep_name',
  429. eq=repname, limit=1)
  430. repo_name = readFile(
  431. path.join(path_rep, "profiles/repo_name")).strip()
  432. if repo_name != repname:
  433. self.printWARNING(
  434. _("Repository '{repo_name}' called '{repname}'"
  435. " in cl_update_rep_name").format(
  436. repo_name=repo_name, repname=repname))
  437. raise UpdateError(_("Failed to update the cache of the {rname} "
  438. "repository").format(rname=repname))
  439. cpu_num = self.clVars.Get('hr_cpu_num')
  440. if repname in OverlayOwnCache(self.clVars):
  441. self.printWARNING(
  442. _("Repository %s has its own cache") % repname.capitalize())
  443. else:
  444. self.startTask(_("Updating the %s repository cache") %
  445. repname.capitalize())
  446. p = self._regenCache_process(egenCache, repname, cpu_num)
  447. if p.failed():
  448. raise UpdateError(_("Failed to update the cache of the {rname} "
  449. "repository").format(rname=repname),
  450. addon=p.read())
  451. return True
  452. def emergeMetadata(self):
  453. """
  454. Выполнить egencache и emerge --metadata
  455. """
  456. emerge = self.get_prog_path("/usr/bin/emerge")
  457. if not emerge:
  458. raise UpdateError(_("The Emerge tool is not found"))
  459. self.addProgress()
  460. p = PercentProgress(emerge, "--ask=n", "--metadata", part=1, atty=True)
  461. for perc in p.progress():
  462. self.setProgress(perc)
  463. if p.failed():
  464. data = p.read()
  465. with open('/var/log/calculate/failed-metadata-%d.log' % time.time(),
  466. 'w') as f:
  467. f.write(data + p.alldata)
  468. raise UpdateError(_("Failed to update metadata"), addon=data)
  469. return True
  470. def _eixUpdateCommand(self, eix_cmd, countRep):
  471. return PercentProgress(eix_cmd, "-F", part=countRep or 1, atty=True)
  472. def eixUpdate(self, repositroies):
  473. """
  474. Выполенине eix-update для репозиторием
  475. eix-update выполнятется только для тех репозиториев, которые
  476. обновлялись, если cl_update_eixsync_force==auto, либо
  477. все, если cl_update_eixupdate_force==force
  478. """
  479. eixupdate = self.get_prog_path("/usr/bin/eix-update")
  480. if not eixupdate:
  481. raise UpdateError(_("The Eix tool is not found"))
  482. self.addProgress()
  483. countRep = len(repositroies)
  484. p = self._eixUpdateCommand(eixupdate, countRep)
  485. for perc in p.progress():
  486. self.setProgress(perc)
  487. if p.failed():
  488. raise UpdateError(_("Failed to update eix cache"), addon=p.read())
  489. return True
  490. def is_binary_pkg(self, pkg, binary=None):
  491. """
  492. Является ли пакет бинарным
  493. """
  494. if binary:
  495. return True
  496. if 'PN' in pkg and pkg['PN'].endswith('-bin'):
  497. return True
  498. if binary is not None:
  499. return binary
  500. if "binary" in pkg and pkg['binary']:
  501. return True
  502. return False
  503. def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
  504. """
  505. Вывод сообщения сборки пакета
  506. """
  507. self.endTask()
  508. _print = self.color_print
  509. one = _print("{0}", num)
  510. two = _print("{0}", max_num)
  511. part = _("({current} of {maximum})").format(current=one, maximum=two)
  512. _print = _print.foreground(Colors.DEFAULT)
  513. if self.is_binary_pkg(pkg, binary):
  514. _colorprint = _print.foreground(Colors.PURPLE)
  515. else:
  516. _colorprint = _print.foreground(Colors.GREEN)
  517. PackageInformation.add_info(pkg)
  518. name = ""
  519. if pkg.info['DESCRIPTION']:
  520. name = _(pkg.info['DESCRIPTION'])
  521. name = name[:1].upper() + name[1:]
  522. if not name:
  523. name = str(pkg)
  524. self.printSUCCESS(
  525. _("{part} {package}").format(part=part, package=name))
  526. self.startTask(
  527. _("Emerging {package}").format(package=_colorprint(str(pkg))))
  528. def _printInstallPackage(self, pkg, binary=False):
  529. """
  530. Вывод сообщения установки пакета
  531. """
  532. self.endTask()
  533. _print = self.color_print
  534. if self.is_binary_pkg(pkg, binary):
  535. _print = _print.foreground(Colors.PURPLE)
  536. else:
  537. _print = _print.foreground(Colors.GREEN)
  538. pkg_key = "{CATEGORY}/{PF}".format(**pkg)
  539. if pkg_key in self.update_map:
  540. self.startTask(_("Installing {pkg} [{oldver}]").format(
  541. pkg=_print(str(pkg)), oldver=self.update_map[pkg_key]))
  542. self.update_map.pop(pkg_key)
  543. else:
  544. self.startTask(_("Installing %s") % (_print(str(pkg))))
  545. def _printFetching(self, fn):
  546. """
  547. Вывод сообщения о скачивании
  548. """
  549. self.endTask()
  550. self.startTask(_("Fetching binary packages"))
  551. def _printUninstallPackage(self, pkg, num=1, max_num=1):
  552. """
  553. Вывод сообщения удаления пакета
  554. """
  555. self.endTask()
  556. _print = self.color_print
  557. if num and max_num:
  558. one = _print("{0}", num)
  559. two = _print("{0}", max_num)
  560. part = _(" ({current} of {maximum})").format(current=one,
  561. maximum=two)
  562. else:
  563. part = ""
  564. _print = _print.foreground(Colors.LIGHT_RED)
  565. self.startTask(
  566. _("Unmerging{part} {package}").format(part=part,
  567. package=_print(str(pkg))))
  568. def emergelike(self, cmd, *params):
  569. """
  570. Запуск команды, которая подразумевает выполнение emerge
  571. """
  572. cmd_path = self.get_prog_path(cmd)
  573. if not cmd_path:
  574. raise UpdateError(_("Failed to find the %s command") % cmd)
  575. with EmergeParser(
  576. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  577. self._startEmerging(emerge)
  578. return True
  579. def revdep_rebuild(self, cmd, *params):
  580. """
  581. Запуск revdep-rebulid
  582. """
  583. cmd_path = self.get_prog_path(cmd)
  584. if not cmd_path:
  585. raise UpdateError(_("Failed to find the %s command") % cmd)
  586. with EmergeParser(
  587. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  588. revdep = RevdepPercentBlock(emerge)
  589. self.addProgress()
  590. revdep.add_observer(self.setProgress)
  591. revdep.action = lambda x: (
  592. self.endTask(), self.startTask(_("Assigning files to packages"))
  593. if "Assign" in revdep else None)
  594. self._startEmerging(emerge)
  595. return True
  596. def _display_pretty_package_list(self, pkglist, remove_list=False):
  597. """
  598. Отобразить список пакетов в "удобочитаемом" виде
  599. """
  600. _print = self.color_print
  601. ebuild_color = TextState.Colors.GREEN
  602. binary_color = TextState.Colors.PURPLE
  603. remove_color = TextState.Colors.LIGHT_RED
  604. flag_map = {"updating":
  605. _print.foreground(TextState.Colors.LIGHT_CYAN)("U"),
  606. "reinstall":
  607. _print.foreground(TextState.Colors.YELLOW)("rR"),
  608. "new":
  609. _print.foreground(TextState.Colors.LIGHT_GREEN)("N"),
  610. "newslot":
  611. _print.foreground(TextState.Colors.LIGHT_GREEN)("NS"),
  612. "downgrading": (
  613. _print.foreground(TextState.Colors.LIGHT_CYAN)("U") +
  614. _print.foreground(TextState.Colors.LIGHT_BLUE)("D"))}
  615. for pkg in sorted([PackageInformation.add_info(x) for x in
  616. pkglist],
  617. key=lambda y: y['CATEGORY/PN']):
  618. install_flag = ""
  619. if remove_list:
  620. pkgcolor = _print.foreground(remove_color)
  621. else:
  622. for flag in flag_map:
  623. if pkg[flag]:
  624. install_flag = "(%s) " % flag_map[flag]
  625. break
  626. if self.is_binary_pkg(pkg):
  627. pkgcolor = _print.foreground(binary_color)
  628. else:
  629. pkgcolor = _print.foreground(ebuild_color)
  630. if pkg.info['DESCRIPTION']:
  631. fullname = "%s " % _(pkg.info['DESCRIPTION'])
  632. fullname = fullname[:1].upper() + fullname[1:]
  633. else:
  634. fullname = ""
  635. shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
  636. if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB":
  637. size = " (%s)" % pkg["SIZE"]
  638. else:
  639. size = ""
  640. mult = _print.bold("*")
  641. self.printDefault(
  642. "&nbsp;{mult} {fullname}{flag}{shortname}{size}".format(
  643. mult=mult, fullname=fullname, shortname=shortname,
  644. size=size,
  645. flag=install_flag))
  646. def _display_install_package(self, emerge, emergelike=False):
  647. """
  648. Отобразить список устанавливаемых пакетов
  649. """
  650. # подробный список пакетов
  651. _print = self.color_print
  652. if emergelike:
  653. self.printPre(str(emerge.install_packages))
  654. else:
  655. pkglist = emerge.install_packages.list
  656. self.printSUCCESS(_print(
  657. _("Listing packages for installation")))
  658. self._display_pretty_package_list(pkglist)
  659. if emerge.install_packages.remove_list:
  660. self.printSUCCESS(_print(
  661. _("Listing packages for removal")))
  662. self._display_pretty_package_list(
  663. emerge.install_packages.remove_list, remove_list=True)
  664. if len(emerge.install_packages.list) > 0:
  665. install_mess = (_("{count} packages will be installed").format(
  666. count=len(emerge.install_packages.list)) + ", ")
  667. else:
  668. install_mess = ""
  669. if str(emerge.download_size) != "0 kB":
  670. self.printSUCCESS(_("{install}{size} will be downloaded").format(
  671. install=install_mess,
  672. size=str(emerge.download_size)))
  673. def _display_remove_list(self, emerge):
  674. """
  675. Отобразить список удаляемых пакетов
  676. """
  677. # подробный список пакетов
  678. if self.clVars.Get('update.cl_update_emergelist_set') == 'on':
  679. self.printPre(self._emerge_translate(
  680. emerge.uninstall_packages.verbose_result))
  681. else:
  682. _print = self.color_print
  683. pkglist = emerge.uninstall_packages.list
  684. self.printSUCCESS(_print.bold(
  685. _("Listing packages for removal")))
  686. self._display_pretty_package_list(pkglist, remove_list=True)
  687. def getCacheOnWorld(self, params, packages, check=False):
  688. """
  689. Получить список обновляемых пакетов @world из кэша
  690. """
  691. if "@world" in packages:
  692. elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge))
  693. if check and (elog.list or elog.remove_list):
  694. self.emerge_cache.drop_cache(
  695. "Some packages was installed or removed")
  696. return params, packages
  697. installed_pkgs = elog.list
  698. new_packages = self.emerge_cache.get_cached_package_list()
  699. if new_packages is not None:
  700. return "-1O", ["=%s" % x for x in new_packages
  701. if not str(x) in installed_pkgs]
  702. return params, packages
  703. def updateCache(self, pkg_list):
  704. """
  705. Обновить кэш. Оставить отметку в emerge.log о том, выполнено действие
  706. premerge
  707. """
  708. self.emerge_cache.set_cache(pkg_list)
  709. elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge))
  710. elog.mark_end_task(),
  711. def mark_schedule(self):
  712. """
  713. Установить отметку о запуске запланированной проверки
  714. """
  715. SystemIni(self.clVars).setVar('update', {'last_check': str(int(time.time()))})
  716. def get_default_emerge_opts(self):
  717. return self.clVars.Get('cl_emerge_default_opts')
  718. def premerge(self, param, *packages):
  719. """
  720. Вывести информацию об обновлении
  721. """
  722. deo = self.clVars.Get('cl_emerge_default_opts')
  723. param, packages = self.getCacheOnWorld(param, packages, check=True)
  724. param = [param, "-pv"]
  725. if not packages:
  726. self.printSUCCESS(_("Installed packages are up to date"))
  727. self.set_need_update(False)
  728. return True
  729. with EmergeParser(EmergeCommand(list(packages), emerge_default_opts=deo,
  730. extra_params=param)) as emerge:
  731. try:
  732. emerge.run()
  733. if "@world" in packages:
  734. if emerge.install_packages.remove_list:
  735. self.emerge_cache.drop_cache(
  736. "List has packages for remove")
  737. elif emerge.install_packages.block_packages:
  738. self.emerge_cache.drop_cache(
  739. "List has block packages")
  740. else:
  741. self.updateCache(emerge.install_packages.list)
  742. if not emerge.install_packages.list:
  743. self.printSUCCESS(_("The system is up to date"))
  744. self.set_need_update(False)
  745. return True
  746. emergelike = self.clVars.Get('cl_update_emergelist_set') == 'on'
  747. self._display_install_package(emerge, emergelike)
  748. if str(emerge.skipped_packages):
  749. self._display_error(emerge.skipped_packages)
  750. except EmergeError:
  751. self.set_need_update(False)
  752. self.emerge_cache.drop_cache("Emerge error")
  753. self._display_install_package(emerge, emergelike=True)
  754. self._display_error(emerge.prepare_error)
  755. raise
  756. if self.clVars.Get('cl_update_pretend_set') == 'on':
  757. # установить кэш: есть обновления
  758. self.set_need_update()
  759. return True
  760. self.set_need_update(False)
  761. answer = self.askConfirm(
  762. _("Would you like to merge these packages?"), "yes")
  763. if answer == "no":
  764. raise KeyboardInterrupt
  765. return "yes"
  766. return True
  767. def set_need_update(self, val=True):
  768. """
  769. Установить флаг: есть обновления
  770. """
  771. if self.clVars.Get('update.cl_update_autocheck_set') == 'off':
  772. val = False
  773. UpdateInfo.set_update_ready(val)
  774. return True
  775. def _emerge_translate(self, s):
  776. """
  777. Перевести текст из emerge
  778. """
  779. return RegexpLocalization('cl_emerge').translate(str(s))
  780. def setUpToDateCache(self):
  781. """
  782. Установить кэш - "нет пакетов для обновления"
  783. """
  784. self.updateCache(PackageList([]))
  785. return True
  786. def _startEmerging(self, emerge):
  787. """
  788. Настроить и выполнить emerge
  789. """
  790. if emerge.install_packages and emerge.install_packages.list:
  791. for pkg in emerge.install_packages.list:
  792. rv = pkg.get('REPLACING_VERSIONS', '')
  793. if rv:
  794. self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \
  795. rv.partition(":")[0]
  796. emerge.command.send("yes\n")
  797. emerge.emerging.add_observer(self._printEmergePackage)
  798. emerge.installing.add_observer(self._printInstallPackage)
  799. emerge.uninstalling.add_observer(self._printUninstallPackage)
  800. emerge.fetching.add_observer(self._printFetching)
  801. def cancel_observing_fetch(fn):
  802. emerge.fetching.clear_observers()
  803. emerge.fetching.add_observer(cancel_observing_fetch)
  804. try:
  805. emerge.run()
  806. except EmergeError:
  807. self.emerge_cache.drop_cache("Emerge error")
  808. if emerge.emerging_error:
  809. self._display_error(emerge.emerging_error.log)
  810. else:
  811. self._display_error(emerge.prepare_error)
  812. raise
  813. def _display_error(self, error):
  814. lines_num = int(self.clVars.Get('update.cl_update_lines_limit'))
  815. error = "<br/>".join(str(error).split('<br/>')[-lines_num:])
  816. self.printPre(self._emerge_translate(error))
  817. def emerge(self, use, param, *packages):
  818. """
  819. Выполнить сборку пакета
  820. """
  821. deo = self.clVars.Get('cl_emerge_default_opts')
  822. if not packages:
  823. packages = [param]
  824. extra_params = None
  825. else:
  826. param, packages = self.getCacheOnWorld(param, packages)
  827. if not packages:
  828. return True
  829. extra_params = [param]
  830. command = EmergeCommand(list(packages), emerge_default_opts=deo,
  831. extra_params=extra_params,
  832. use=use)
  833. if self.clVars.Get('cl_chroot_path') != '/':
  834. command = Chroot(self.clVars.Get('cl_chroot_path'), command)
  835. with EmergeParser(command) as emerge:
  836. try:
  837. emerge.question.action = lambda x: False
  838. emerge.run()
  839. if not emerge.install_packages.list:
  840. return True
  841. except EmergeError:
  842. self.emerge_cache.drop_cache("Emerge error")
  843. self._display_error(emerge.prepare_error)
  844. raise
  845. self._startEmerging(emerge)
  846. return True
  847. def depclean(self):
  848. """
  849. Выполнить очистку системы от лишних пакетов
  850. """
  851. deo = self.get_default_emerge_opts()
  852. emerge = None
  853. try:
  854. emerge = EmergeParser(EmergeCommand(["--depclean"],
  855. emerge_default_opts=deo))
  856. outdated_kernel = False
  857. try:
  858. emerge.question.action = lambda x: False
  859. emerge.run()
  860. if not emerge.uninstall_packages.list:
  861. UpdateInfo(self.clVars).outdated_kernel = False
  862. return True
  863. kernel_pkg = self.clVars.Get('cl_update_kernel_pkg')
  864. if any(("%s-%s" % (x['CATEGORY/PN'], x['PVR'])) == kernel_pkg
  865. for x in emerge.uninstall_packages.list):
  866. pkglist = [
  867. "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in
  868. emerge.uninstall_packages.list
  869. if ("%s-%s" % (x['CATEGORY/PN'],
  870. x['PVR'])) != kernel_pkg]
  871. emerge.command.send('n\n')
  872. emerge.close()
  873. emerge = None
  874. if not pkglist:
  875. UpdateInfo(self.clVars).outdated_kernel = True
  876. return True
  877. emerge = EmergeParser(
  878. EmergeCommand(pkglist,
  879. extra_params=["--unmerge", '--ask=y'],
  880. emerge_default_opts=deo))
  881. emerge.question.action = lambda x: False
  882. emerge.run()
  883. outdated_kernel = True
  884. else:
  885. outdated_kernel = False
  886. self._display_remove_list(emerge)
  887. except EmergeError:
  888. self._display_error(emerge.prepare_error)
  889. raise
  890. if (self.askConfirm(
  891. _("Would you like to unmerge these unused packages "
  892. "(recommended)?")) != 'yes'):
  893. return True
  894. UpdateInfo(self.clVars).outdated_kernel = outdated_kernel
  895. self._startEmerging(emerge)
  896. finally:
  897. if emerge:
  898. emerge.close()
  899. return True
  900. def update_task(self, task_name):
  901. """
  902. Декоратор для добавления меток запуска и останова задачи
  903. """
  904. def decor(f):
  905. def wrapper(*args, **kwargs):
  906. logger = EmergeLog(EmergeLogNamedTask(task_name))
  907. logger.mark_begin_task()
  908. ret = f(*args, **kwargs)
  909. if ret:
  910. logger.mark_end_task()
  911. return ret
  912. return wrapper
  913. return decor
  914. def migrateCacheRepository(self, url, branch, storage):
  915. """
  916. Перенести репозиторий из кэша в локальный
  917. """
  918. rep = storage.get_repository(url, branch)
  919. if rep:
  920. rep.storage = storage.storages[0]
  921. self.clVars.Invalidate('cl_update_profile_storage')
  922. return True
  923. def reconfigureProfileVars(self, profile_dv, chroot):
  924. """
  925. Синхронизировать репозитории
  926. """
  927. dv = self.clVars
  928. try:
  929. if not profile_dv:
  930. raise UpdateError(
  931. _("Failed to use the new profile. Try again."))
  932. for var_name in ('cl_update_rep_path',
  933. 'cl_update_rep_url',
  934. 'cl_update_rep_name',
  935. 'cl_update_branch',
  936. 'cl_update_binhost_list',
  937. 'cl_update_binhost_unstable_list',
  938. 'cl_update_binhost_stable_set',
  939. 'cl_update_binhost_stable_opt_set',
  940. 'cl_update_branch_name',
  941. 'cl_profile_system',
  942. 'cl_update_rep'):
  943. dv.Set(var_name, profile_dv.Get(var_name), force=True)
  944. dv.Set('cl_chroot_path', chroot, force=True)
  945. except DataVarsError as e:
  946. print str(e)
  947. raise UpdateError(_("Wrong profile"))
  948. return True
  949. def setProfile(self, profile_shortname):
  950. profile = self.clVars.Select('cl_update_profile_path',
  951. where='cl_update_profile_shortname',
  952. eq=profile_shortname, limit=1)
  953. if not profile:
  954. raise UpdateError(_("Failed to determine profile %s") %
  955. self.clVars.Get('cl_update_profile_system'))
  956. profile_path = path.relpath(profile, '/etc/portage')
  957. try:
  958. profile_file = '/etc/portage/make.profile'
  959. if not path.exists(
  960. path.join(path.dirname(profile_file), profile_path)):
  961. raise UpdateError(
  962. _("Failed to set the profile: %s") % _("Profile not found"))
  963. for rm_fn in filter(path.lexists,
  964. ('/etc/make.profile',
  965. '/etc/portage/make.profile')):
  966. os.unlink(rm_fn)
  967. os.symlink(profile_path, profile_file)
  968. except (OSError, IOError) as e:
  969. raise UpdateError(_("Failed to set the profile: %s") % str(e))
  970. return True
  971. def applyProfileTemplates(self, useClt=None, cltFilter=False,
  972. useDispatch=True, action="merge"):
  973. """
  974. Наложить шаблоны из профиля
  975. """
  976. from calculate.lib.cl_template import TemplatesError, ProgressTemplate
  977. dv = DataVarsUpdate()
  978. try:
  979. dv.importUpdate()
  980. dv.flIniFile()
  981. dv.Set('cl_action', action, force=True)
  982. dv.Set('cl_templates_locate',
  983. self.clVars.Get('cl_update_templates_locate'))
  984. dv.Set("cl_chroot_path", '/', True)
  985. dv.Set("cl_root_path", '/', True)
  986. for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
  987. "update.cl_update_world"):
  988. dv.Set(copyvar, self.clVars.Get(copyvar), True)
  989. # определение каталогов содержащих шаблоны
  990. useClt = useClt in ("on", True)
  991. self.addProgress()
  992. nullProgress = lambda *args, **kw: None
  993. dispatch = self.dispatchConf if useDispatch else None
  994. clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt,
  995. cltFilter=cltFilter,
  996. printSUCCESS=self.printSUCCESS,
  997. printWARNING=self.printWARNING,
  998. askConfirm=self.askConfirm,
  999. dispatchConf=dispatch,
  1000. printERROR=self.printERROR)
  1001. try:
  1002. clTempl.applyTemplates()
  1003. if clTempl.hasError():
  1004. if clTempl.getError():
  1005. raise TemplatesError(clTempl.getError())
  1006. finally:
  1007. if clTempl:
  1008. if clTempl.cltObj:
  1009. clTempl.cltObj.closeFiles()
  1010. clTempl.closeFiles()
  1011. finally:
  1012. dv.close()
  1013. return True
  1014. def cleanpkg(self):
  1015. """
  1016. Очистить PKGDIR и DISTFILES в текущей системе
  1017. """
  1018. portdirs = ([self.clVars.Get('cl_portdir')] +
  1019. self.clVars.Get('cl_portdir_overlay'))
  1020. pkgfiles = get_packages_files_directory(*portdirs)
  1021. distdirfiles = get_manifest_files_directory(*portdirs)
  1022. distdir = self.clVars.Get('install.cl_distfiles_path')
  1023. pkgdir = self.clVars.Get('cl_pkgdir')
  1024. logger = log("update_cleanpkg.log",
  1025. filename="/var/log/calculate/update_cleanpkg.log",
  1026. formatter="%(asctime)s - %(clean)s - %(message)s")
  1027. return self._cleanpkg(
  1028. distdir, pkgdir, distdirfiles, pkgfiles, logger)
  1029. def _update_binhost_packages(self):
  1030. os.system('/usr/sbin/emaint binhost -f &>/dev/null')
  1031. def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger):
  1032. """
  1033. Общий алгоритм очистки distfiles и pkgdir от устаревших пакетов
  1034. """
  1035. skip_files = ["/metadata.dtd", "/Packages"]
  1036. try:
  1037. if self.clVars.Get('client.os_remote_auth'):
  1038. skip_files += ['portage_lockfile']
  1039. except DataVarsError:
  1040. pass
  1041. for cleantype, filelist in (
  1042. ("packages",
  1043. get_remove_list(pkgdir, list(pkgfiles), depth=4)),
  1044. ("distfiles",
  1045. get_remove_list(distdir, list(distdirfiles), depth=1))):
  1046. removelist = []
  1047. for fn in filelist:
  1048. try:
  1049. if not any(fn.endswith(x) for x in skip_files):
  1050. os.unlink(fn)
  1051. removelist.append(path.basename(fn))
  1052. except OSError:
  1053. pass
  1054. removelist_str = ",".join(removelist)
  1055. if removelist_str:
  1056. logger.info(removelist_str, extra={'clean': cleantype})
  1057. if cleantype == "packages":
  1058. try:
  1059. self._update_binhost_packages()
  1060. for dn in listDirectory(pkgdir, fullPath=True):
  1061. if path.isdir(dn) and not listDirectory(dn):
  1062. os.rmdir(dn)
  1063. except OSError:
  1064. pass
  1065. return True
  1066. def updateSetupCache(self):
  1067. cache = SetupCache(self.clVars)
  1068. cache.update(force=True)
  1069. return True
  1070. def check_binhost(self, write_binhost=True):
  1071. """
  1072. Проверить, что доступен хотя бы один из binhost'ов
  1073. :return:
  1074. """
  1075. hosts = self.clVars.Get("update.cl_update_binhost_host")
  1076. datas = self.clVars.Get("update.cl_update_binhost_revisions")
  1077. bin_cache_fn = pathJoin(self.clVars.Get('cl_chroot_path'),
  1078. LayeredIni.IniPath.Grp)
  1079. if not hosts:
  1080. self.clVars.Delete('cl_update_binhost', location="system")
  1081. try:
  1082. if path.exists(bin_cache_fn):
  1083. os.unlink(bin_cache_fn)
  1084. except OSError:
  1085. raise UpdateError(
  1086. _("Failed to clear ini cache of binary repository"))
  1087. raise UpdateError(_("Failed to find the server with "
  1088. "appropriate updates"))
  1089. else:
  1090. with writeFile(bin_cache_fn) as f:
  1091. f.write(datas[0].strip()+"\n")
  1092. if write_binhost:
  1093. if hosts[0] != self.clVars.Get('update.cl_update_binhost'):
  1094. self.refresh_binhost = True
  1095. self.clVars.Write('cl_update_binhost', hosts[0], location="system")
  1096. new_ts = self.clVars.Get("update.cl_update_binhost_timestamp")
  1097. if new_ts:
  1098. new_ts = new_ts[0]
  1099. if new_ts.isdigit():
  1100. ini = SystemIni(self.clVars)
  1101. ini.setVar('update', {'last_update': new_ts})
  1102. if self.clVars.Get('cl_action') == 'sync':
  1103. value = self.clVars.GetBool('cl_update_binhost_stable_set')
  1104. new_value = self.clVars.GetBool('cl_update_binhost_stable_opt_set')
  1105. if value != new_value:
  1106. self.clVars.Write(
  1107. 'cl_update_binhost_stable_set',
  1108. self.clVars.Get('update.cl_update_binhost_stable_opt_set'),
  1109. location="system")
  1110. return True
  1111. def message_binhost_changed(self):
  1112. if self.refresh_binhost:
  1113. self.printWARNING(_("Update server was changed to %s") %
  1114. self.clVars.Get('cl_update_binhost'))
  1115. else:
  1116. self.printSUCCESS(_("Update server %s") %
  1117. self.clVars.Get('cl_update_binhost'))
  1118. return True
  1119. def update_binhost_list(self):
  1120. """
  1121. Обновить список binhost'ов после обновления до master веток
  1122. :return:
  1123. """
  1124. dv = DataVarsUpdate()
  1125. try:
  1126. dv.importUpdate()
  1127. dv.flIniFile()
  1128. changes = False
  1129. for varname in ('update.cl_update_binhost_list',
  1130. 'update.cl_update_binhost_unstable_list',
  1131. 'update.cl_update_binhost_timestamp_path',
  1132. 'cl_update_binhost_revision_path'):
  1133. new_value = dv.Get(varname)
  1134. old_value = self.clVars.Get(varname)
  1135. if new_value != old_value:
  1136. changes = True
  1137. self.clVars.Set(varname, new_value, force=True)
  1138. if not changes:
  1139. self.clVars.Delete('cl_update_binhost', location="system")
  1140. raise UpdateError(_("Failed to find the server with "
  1141. "appropriate updates"))
  1142. except DataVarsError:
  1143. self.clVars.Delete('cl_update_binhost', location="system")
  1144. raise UpdateError(_("Failed to find the server with "
  1145. "appropriate updates"))
  1146. return True
  1147. def drop_binhosts(self, dv):
  1148. """
  1149. Обновление до master веток
  1150. """
  1151. branch = dv.Get('update.cl_update_branch')
  1152. revs = [
  1153. branch for x in dv.Get('update.cl_update_rep_name')
  1154. ]
  1155. dv.Set('update.cl_update_branch_name', revs)
  1156. dv.Invalidate('update.cl_update_rep_rev')
  1157. return True