You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1289 lines
52 KiB

  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 Mir Calculate. http://www.calculate-linux.org
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from functools import wraps
  16. import random
  17. import sys
  18. from os import path
  19. import os
  20. import time
  21. from calculate.core.server.gen_pid import search_worked_process
  22. from calculate.core.setup_cache import Cache as SetupCache
  23. from calculate.core.server.func import MethodsInterface
  24. from calculate.lib.cl_template import SystemIni, LayeredIni
  25. from calculate.lib.datavars import DataVarsError
  26. from calculate.lib.utils.tools import AddonError
  27. from calculate.lib.utils.colortext.palette import TextState
  28. from calculate.lib.utils.colortext import get_color_print
  29. from calculate.update.emerge_parser import RevdepPercentBlock
  30. from calculate.update.datavars import DataVarsUpdate
  31. from calculate.update.update_info import UpdateInfo
  32. from calculate.lib.utils.binhosts import fetch_packages
  33. from calculate.lib.cl_log import log
  34. import re
  35. import shutil
  36. from collections import MutableSet
  37. from calculate.lib.utils.git import Git, GitError, MTimeKeeper, NotGitError
  38. from calculate.lib.utils.portage import (Layman, EmergeLog,
  39. EmergeLogNamedTask,
  40. PackageInformation,
  41. get_packages_files_directory,
  42. get_manifest_files_directory,
  43. get_remove_list)
  44. from calculate.lib.utils.text import _u8
  45. Colors = TextState.Colors
  46. from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
  47. PercentProgress, process, getRunCommands,
  48. readFile, listDirectory, pathJoin,
  49. writeFile)
  50. import emerge_parser
  51. import logging
  52. from emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
  53. EmergeCache, Chroot)
  54. from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
  55. RegexpLocalization, _)
  56. setLocalTranslate('cl_update3', sys.modules[__name__])
  57. __ = getLazyLocalTranslate(_)
  58. class UpdateError(AddonError):
  59. """Update Error"""
  60. class OverlayOwnCache(MutableSet):
  61. """
  62. Сет оверлеев с интегрированным кэшем
  63. """
  64. def __init__(self, dv=None):
  65. self.dv = dv
  66. def __get_overlays(self):
  67. own_cache_value = SystemIni(self.dv).getVar('system', 'own_cache') or ""
  68. return [x.strip() for x in own_cache_value.split(',') if x.strip()]
  69. def __write_overlays(self, overlays):
  70. if not overlays:
  71. SystemIni(self.dv).delVar('system', 'own_cache')
  72. else:
  73. SystemIni(self.dv).setVar('system',
  74. {'own_cache': ",".join(overlays)})
  75. def __contains__(self, item):
  76. return item in self.__get_overlays()
  77. def __iter__(self):
  78. return iter(self.__get_overlays())
  79. def __len__(self):
  80. return len(self.__get_overlays())
  81. def __append_value(self, overlays, value):
  82. if value not in overlays:
  83. overlays.append(value)
  84. self.__write_overlays(overlays)
  85. def add(self, value):
  86. overlays = self.__get_overlays()
  87. self.__append_value(overlays, value)
  88. def discard(self, value):
  89. overlays = self.__get_overlays()
  90. if value in overlays:
  91. overlays.remove(value)
  92. self.__write_overlays(overlays)
  93. def variable_module(var_env):
  94. def variable_module_decor(f):
  95. @wraps(f)
  96. def wrapper(self, *args, **kw):
  97. old_env = self.clVars.defaultModule
  98. try:
  99. self.clVars.defaultModule = var_env
  100. return f(self, *args, **kw)
  101. finally:
  102. self.clVars.defaultModule = old_env
  103. return wrapper
  104. return variable_module_decor
  105. class Update(MethodsInterface):
  106. """Основной объект для выполнения действий связанных с обновлением системы
  107. """
  108. def init(self):
  109. commandLog = path.join(self.clVars.Get('core.cl_log_path'),
  110. 'lastcommand.log')
  111. emerge_parser.CommandExecutor.logfile = commandLog
  112. self.color_print = get_color_print()
  113. self.emerge_cache = EmergeCache()
  114. if self.clVars.Get('cl_env_debug_set') == 'off':
  115. EmergeCache.logger.logger.setLevel(logging.WARNING)
  116. self.emerge_cache.check_list = (
  117. self.emerge_cache.check_list +
  118. map(emerge_parser.GitCheckvalue,
  119. self.clVars.Get('update.cl_update_rep_path')))
  120. self.update_map = {}
  121. self.refresh_binhost = False
  122. self.pkgnum = None
  123. self.pkgnummax = None
  124. def get_prog_path(self, program_name):
  125. return getProgPath(program_name)
  126. def _syncRepository(self, name, url, rpath, revision,
  127. cb_progress=None, clean=False, notask=False):
  128. """
  129. Синхронизировать репозитори
  130. """
  131. dv = self.clVars
  132. git = Git()
  133. info_outdated = False
  134. old_dir = "%s.old" % git._gitDir(rpath)
  135. if path.exists(old_dir):
  136. clean = True
  137. try:
  138. self.stash_cache(rpath, name)
  139. if not git.checkExistsRep(rpath):
  140. if not notask:
  141. self.startTask(_("Syncing the {rep} repository").format(
  142. rep=name.capitalize()))
  143. self.addProgress()
  144. git.cloneTagRepository(url, rpath, revision,
  145. cb_progress=cb_progress)
  146. info_outdated = True
  147. else:
  148. cr = ""
  149. try:
  150. need_update = False
  151. tag_cr = git.getCommit(rpath, revision)
  152. cr = git.getCurrentCommit(rpath)
  153. ref_type = git.reference_type(rpath, revision)
  154. if tag_cr != cr or ref_type == Git.Reference.Branch:
  155. need_update = True
  156. elif clean:
  157. status = git.getStatusInfo(rpath)
  158. if not status or status['files']:
  159. need_update = True
  160. except GitError:
  161. need_update = True
  162. if need_update:
  163. if not notask:
  164. self.startTask(_("Syncing the {rep} repository").format(
  165. rep=name.capitalize()))
  166. self.addProgress()
  167. git.updateTagRepository(url, rpath, revision,
  168. cb_progress=cb_progress,
  169. clean=clean)
  170. new_cr = git.getCurrentCommit(rpath)
  171. if new_cr != cr:
  172. info_outdated = True
  173. if info_outdated:
  174. self.raiseOutdate()
  175. dv.Set('cl_update_outdate_set', 'on', force=True)
  176. finally:
  177. self.unstash_cache(rpath, name)
  178. return True
  179. def raiseOutdate(self):
  180. self.clVars.Set('cl_update_outdate_set', 'on', force=True)
  181. def setAutocheckParams(self, status, interval, update_other, cleanpkg):
  182. """
  183. Настроить параметры автопроверки обновлений
  184. """
  185. onoff = lambda x: "on" if x else "off"
  186. self.clVars.Write('cl_update_autocheck_set', onoff(status), True)
  187. self.clVars.Write('cl_update_autocheck_interval', interval, True)
  188. self.clVars.Write('cl_update_other_set', onoff(update_other), True)
  189. self.clVars.Write('cl_update_cleanpkg_set', onoff(cleanpkg), True)
  190. if not status:
  191. UpdateInfo.set_update_ready(False)
  192. return True
  193. def checkSchedule(self, interval, status):
  194. """
  195. Проверить по расписанию необходимость запуска команды
  196. """
  197. if not status:
  198. self.printWARNING(_("Updates autocheck is not enabled"))
  199. return False
  200. last_check = SystemIni(self.clVars).getVar('system', 'last_check') or ""
  201. re_interval = re.compile("^(\d+)\s*(hours?|days?|weeks?)?", re.I)
  202. interval_match = re_interval.search(interval)
  203. MINUTE = 60
  204. HOUR = MINUTE * 60
  205. DAY = HOUR * 24
  206. WEEK = DAY * 7
  207. if interval_match:
  208. if interval_match.group(2):
  209. suffix_map = {'h': HOUR, 'd': DAY, 'w': WEEK}
  210. k = suffix_map.get(interval_match.group(2).lower()[0], HOUR)
  211. else:
  212. k = HOUR
  213. est = int(interval_match.group(1)) * k
  214. else:
  215. est = 3 * HOUR
  216. if last_check:
  217. if last_check.isdigit():
  218. if (time.time() - int(last_check)) < (est - 10 * MINUTE):
  219. self.printWARNING(_("Please wait for the update time"))
  220. return False
  221. self.mark_schedule()
  222. return True
  223. def checkRun(self, wait_update):
  224. """
  225. Проверить повторный запуск
  226. """
  227. update_running = lambda: any(os.getpid() != x
  228. for x in
  229. search_worked_process('update', dv))
  230. dv = self.clVars
  231. if update_running():
  232. if not wait_update:
  233. raise UpdateError(_("Update is already running. "
  234. "Try to run later."))
  235. else:
  236. self.startTask(_("Waiting for another update to be complete"))
  237. while update_running():
  238. self.pauseProcess()
  239. while update_running():
  240. time.sleep(0.3)
  241. self.resumeProcess()
  242. time.sleep(random.random() * 3)
  243. self.endTask()
  244. if self.clVars.Get('cl_chroot_status') == 'off':
  245. emerge_running = lambda: any("/usr/bin/emerge" in x
  246. for x in getRunCommands(True))
  247. if emerge_running():
  248. if not wait_update:
  249. raise UpdateError(_("Emerge is running. "
  250. "Try to run later."))
  251. else:
  252. self.startTask(_("Waiting for emerge to be complete"))
  253. while emerge_running():
  254. time.sleep(1)
  255. self.endTask()
  256. return True
  257. @variable_module("update")
  258. def trimRepositories(self, repname):
  259. """
  260. Синхронизировать репозитории
  261. """
  262. dv = self.clVars
  263. rpath = \
  264. dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1)
  265. git = Git()
  266. self.addProgress()
  267. git.trimRepository(rpath, cb_progress=self.setProgress)
  268. return True
  269. @variable_module("update")
  270. def syncRepositories(self, repname, fallback_sync=False,
  271. clean_on_error=True):
  272. """
  273. Синхронизировать репозитории
  274. """
  275. dv = self.clVars
  276. check_status = dv.GetBool('update.cl_update_check_rep_set')
  277. url, rpath, revision = (
  278. dv.Select(["cl_update_rep_url", "cl_update_rep_path",
  279. "cl_update_rep_rev"],
  280. where="cl_update_rep_name", eq=repname, limit=1))
  281. if not url or not rpath:
  282. raise UpdateError(_("Configuration variables for repositories "
  283. "are not setup"))
  284. git = Git()
  285. if not git.checkUrl(url):
  286. raise UpdateError(_("Git %s is unavailable") % url)
  287. chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
  288. if chroot_path == '/':
  289. rpath_orig = rpath
  290. else:
  291. rpath_orig = rpath[len(chroot_path):]
  292. mtime = MTimeKeeper(path.join(rpath, "profiles/updates"))
  293. mtime.save()
  294. try:
  295. if clean_on_error:
  296. try:
  297. layman = Layman(dv.Get('cl_update_layman_installed'),
  298. dv.Get('cl_update_layman_make'),
  299. dv.Get('cl_update_layman_conf'),
  300. prefix=chroot_path)
  301. if repname not in ("portage", "gentoo"):
  302. layman.add(repname, url, rpath_orig)
  303. if not self._syncRepository(repname, url, rpath, revision,
  304. cb_progress=self.setProgress,
  305. clean=check_status,
  306. notask=fallback_sync):
  307. return "skip"
  308. return True
  309. except GitError as e:
  310. if not isinstance(e, NotGitError):
  311. if e.addon:
  312. self.printWARNING(str(e.addon))
  313. self.printWARNING(str(e))
  314. self.endTask(False)
  315. self.startTask(
  316. _("Re-fetching the {name} repository").format(
  317. name=repname))
  318. self.addProgress()
  319. rpath_new = "%s_new" % rpath
  320. try:
  321. self._syncRepository(repname, url, rpath_new, revision,
  322. cb_progress=self.setProgress,
  323. clean=check_status,
  324. notask=fallback_sync)
  325. removeDir(rpath)
  326. shutil.move(rpath_new, rpath)
  327. except OSError as e:
  328. raise UpdateError(_("Failed to modify the "
  329. "{repname} repository").format(
  330. repname=repname) + _(": ") + str(e))
  331. finally:
  332. if path.exists(rpath_new):
  333. removeDir(rpath_new)
  334. else:
  335. if not self._syncRepository(repname, url, rpath, revision,
  336. clean=check_status):
  337. return "skip"
  338. layman = Layman(dv.Get('cl_update_layman_installed'),
  339. dv.Get('cl_update_layman_make'),
  340. dv.Get('cl_update_layman_conf'),
  341. prefix=chroot_path)
  342. if repname not in ("portage", "gentoo"):
  343. layman.add(repname, url, rpath_orig)
  344. finally:
  345. mtime.restore()
  346. return True
  347. metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
  348. def stash_cache(self, rpath, name):
  349. """
  350. Спрятать кэш
  351. """
  352. if name in ("portage", "gentoo"):
  353. return
  354. if not name in OverlayOwnCache(self.clVars):
  355. for cachename in self.metadata_cache_names:
  356. cachedir = path.join(rpath, cachename)
  357. if path.exists(cachedir):
  358. try:
  359. cachedir_s = path.join(path.dirname(rpath),
  360. path.basename(
  361. cachename) + ".stash")
  362. if path.exists(cachedir_s):
  363. removeDir(cachedir_s)
  364. shutil.move(cachedir, cachedir_s)
  365. except BaseException as e:
  366. pass
  367. def unstash_cache(self, rpath, name):
  368. """
  369. Извлеч кэш
  370. """
  371. if name in ("portage", "gentoo"):
  372. return
  373. cachenames = self.metadata_cache_names
  374. if not name in OverlayOwnCache(self.clVars):
  375. if any(path.exists(path.join(rpath, x)) for x in cachenames):
  376. for cachename in cachenames:
  377. cachedir_s = path.join(path.dirname(rpath),
  378. path.basename(cachename) + ".stash")
  379. if path.exists(cachedir_s):
  380. try:
  381. removeDir(cachedir_s)
  382. except BaseException as e:
  383. pass
  384. OverlayOwnCache(self.clVars).add(name)
  385. else:
  386. for cachename in cachenames:
  387. cachedir = path.join(rpath, cachename)
  388. cachedir_s = path.join(path.dirname(rpath),
  389. path.basename(cachename) + ".stash")
  390. if path.exists(cachedir_s):
  391. try:
  392. shutil.move(cachedir_s, cachedir)
  393. except BaseException as e:
  394. pass
  395. else:
  396. if all(not path.exists(path.join(rpath, x)) for x in cachenames):
  397. OverlayOwnCache(self.clVars).discard(name)
  398. def syncLaymanRepository(self, repname):
  399. """
  400. Обновить репозиторий через layman
  401. """
  402. layman = self.get_prog_path('/usr/bin/layman')
  403. if not layman:
  404. raise UpdateError(_("The Layman tool is not found"))
  405. rpath = self.clVars.Select('cl_update_other_rep_path',
  406. where='cl_update_other_rep_name', eq=repname,
  407. limit=1)
  408. laymanname = path.basename(rpath)
  409. self.stash_cache(rpath, laymanname)
  410. try:
  411. if Git.is_git(rpath):
  412. self.addProgress()
  413. p = PercentProgress(layman, "-s", laymanname, part=1, atty=True)
  414. for perc in p.progress():
  415. self.setProgress(perc)
  416. else:
  417. p = process(layman, "-s", repname, stderr=STDOUT)
  418. if p.failed():
  419. raise UpdateError(
  420. _("Failed to update the {rname} repository").format(
  421. rname=repname),
  422. addon=p.read())
  423. finally:
  424. self.unstash_cache(rpath, laymanname)
  425. return True
  426. def _regenCache_process(self, progname, repname, cpu_num):
  427. return process(progname, "--repo=%s" % repname, "--update",
  428. "--jobs=%s" % cpu_num, stderr=STDOUT)
  429. def regenCache(self, repname):
  430. """
  431. Обновить кэш метаданных репозитория
  432. """
  433. egenCache = self.get_prog_path('/usr/bin/egencache')
  434. if not egenCache:
  435. raise UpdateError(_("The Portage tool is not found"))
  436. if repname in self.clVars.Get('cl_update_rep_name'):
  437. path_rep = self.clVars.Select('cl_update_rep_path',
  438. where='cl_update_rep_name',
  439. eq=repname, limit=1)
  440. repo_name = readFile(
  441. path.join(path_rep, "profiles/repo_name")).strip()
  442. if repo_name != repname:
  443. self.printWARNING(
  444. _("Repository '{repo_name}' called '{repname}'"
  445. " in cl_update_rep_name").format(
  446. repo_name=repo_name, repname=repname))
  447. raise UpdateError(_("Failed to update the cache of the {rname} "
  448. "repository").format(rname=repname))
  449. cpu_num = self.clVars.Get('hr_cpu_num')
  450. if repname in OverlayOwnCache(self.clVars):
  451. self.printWARNING(
  452. _("Repository %s has its own cache") % repname.capitalize())
  453. else:
  454. self.startTask(_("Updating the %s repository cache") %
  455. repname.capitalize())
  456. p = self._regenCache_process(egenCache, repname, cpu_num)
  457. if p.failed():
  458. raise UpdateError(_("Failed to update the cache of the {rname} "
  459. "repository").format(rname=repname),
  460. addon=p.read())
  461. return True
  462. def emergeMetadata(self):
  463. """
  464. Выполнить egencache и emerge --metadata
  465. """
  466. emerge = self.get_prog_path("/usr/bin/emerge")
  467. if not emerge:
  468. raise UpdateError(_("The Emerge tool is not found"))
  469. self.addProgress()
  470. p = PercentProgress(emerge, "--ask=n", "--metadata", part=1, atty=True)
  471. for perc in p.progress():
  472. self.setProgress(perc)
  473. if p.failed():
  474. data = p.read()
  475. with open('/var/log/calculate/failed-metadata-%d.log' % time.time(),
  476. 'w') as f:
  477. f.write(data + p.alldata)
  478. raise UpdateError(_("Failed to update metadata"), addon=data)
  479. return True
  480. def _eixUpdateCommand(self, eix_cmd, countRep):
  481. return PercentProgress(eix_cmd, "-F", part=countRep or 1, atty=True)
  482. def eixUpdate(self, repositroies):
  483. """
  484. Выполенине eix-update для репозиторием
  485. eix-update выполнятется только для тех репозиториев, которые
  486. обновлялись, если cl_update_eixsync_force==auto, либо
  487. все, если cl_update_eixupdate_force==force
  488. """
  489. eixupdate = self.get_prog_path("/usr/bin/eix-update")
  490. if not eixupdate:
  491. raise UpdateError(_("The Eix tool is not found"))
  492. self.addProgress()
  493. countRep = len(repositroies)
  494. p = self._eixUpdateCommand(eixupdate, countRep)
  495. for perc in p.progress():
  496. self.setProgress(perc)
  497. if p.failed():
  498. raise UpdateError(_("Failed to update eix cache"), addon=p.read())
  499. return True
  500. def is_binary_pkg(self, pkg, binary=None):
  501. """
  502. Является ли пакет бинарным
  503. """
  504. if binary:
  505. return True
  506. if 'PN' in pkg and pkg['PN'].endswith('-bin'):
  507. return True
  508. if binary is not None:
  509. return binary
  510. if "binary" in pkg and pkg['binary']:
  511. return True
  512. return False
  513. def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
  514. """
  515. Вывод сообщения сборки пакета
  516. """
  517. self.endTask()
  518. _print = self.color_print
  519. if self.pkgnum is not None:
  520. num = self.pkgnum
  521. if self.pkgnummax is not None:
  522. max_num = self.pkgnummax
  523. one = _print("{0}", num)
  524. two = _print("{0}", max_num)
  525. part = _("({current} of {maximum})").format(current=one, maximum=two)
  526. _print = _print.foreground(Colors.DEFAULT)
  527. if self.is_binary_pkg(pkg, binary):
  528. _colorprint = _print.foreground(Colors.PURPLE)
  529. else:
  530. _colorprint = _print.foreground(Colors.GREEN)
  531. PackageInformation.add_info(pkg)
  532. name = ""
  533. if pkg.info['DESCRIPTION']:
  534. name = _(pkg.info['DESCRIPTION'])
  535. name = name[:1].upper() + name[1:]
  536. if not name:
  537. name = str(pkg)
  538. self.printSUCCESS(
  539. _("{part} {package}").format(part=part, package=name))
  540. self.startTask(
  541. _("Emerging {package}").format(package=_colorprint(str(pkg))))
  542. def _printInstallPackage(self, pkg, binary=False):
  543. """
  544. Вывод сообщения установки пакета
  545. """
  546. self.endTask()
  547. _print = self.color_print
  548. if self.is_binary_pkg(pkg, binary):
  549. _print = _print.foreground(Colors.PURPLE)
  550. else:
  551. _print = _print.foreground(Colors.GREEN)
  552. pkg_key = "{CATEGORY}/{PF}".format(**pkg)
  553. if pkg_key in self.update_map:
  554. self.startTask(_("Installing {pkg} [{oldver}]").format(
  555. pkg=_print(str(pkg)), oldver=self.update_map[pkg_key]))
  556. self.update_map.pop(pkg_key)
  557. else:
  558. self.startTask(_("Installing %s") % (_print(str(pkg))))
  559. def _printFetching(self, fn):
  560. """
  561. Вывод сообщения о скачивании
  562. """
  563. self.endTask()
  564. self.startTask(_("Fetching binary packages"))
  565. def _printUninstallPackage(self, pkg, num=1, max_num=1):
  566. """
  567. Вывод сообщения удаления пакета
  568. """
  569. self.endTask()
  570. _print = self.color_print
  571. if num and max_num:
  572. one = _print("{0}", num)
  573. two = _print("{0}", max_num)
  574. part = _(" ({current} of {maximum})").format(current=one,
  575. maximum=two)
  576. else:
  577. part = ""
  578. _print = _print.foreground(Colors.LIGHT_RED)
  579. self.startTask(
  580. _("Unmerging{part} {package}").format(part=part,
  581. package=_print(str(pkg))))
  582. def emergelike(self, cmd, *params):
  583. """
  584. Запуск команды, которая подразумевает выполнение emerge
  585. """
  586. cmd_path = self.get_prog_path(cmd)
  587. if not cmd_path:
  588. raise UpdateError(_("Failed to find the %s command") % cmd)
  589. with EmergeParser(
  590. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  591. self._startEmerging(emerge)
  592. return True
  593. def revdep_rebuild(self, cmd, *params):
  594. """
  595. Запуск revdep-rebulid
  596. """
  597. cmd_path = self.get_prog_path(cmd)
  598. if not cmd_path:
  599. raise UpdateError(_("Failed to find the %s command") % cmd)
  600. with EmergeParser(
  601. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  602. revdep = RevdepPercentBlock(emerge)
  603. self.addProgress()
  604. revdep.add_observer(self.setProgress)
  605. revdep.action = lambda x: (
  606. self.endTask(), self.startTask(_("Assigning files to packages"))
  607. if "Assign" in revdep else None)
  608. self._startEmerging(emerge)
  609. return True
  610. def _display_pretty_package_list(self, pkglist, remove_list=False):
  611. """
  612. Отобразить список пакетов в "удобочитаемом" виде
  613. """
  614. _print = self.color_print
  615. ebuild_color = TextState.Colors.GREEN
  616. binary_color = TextState.Colors.PURPLE
  617. remove_color = TextState.Colors.LIGHT_RED
  618. flag_map = {"updating":
  619. _print.foreground(TextState.Colors.LIGHT_CYAN)("U"),
  620. "reinstall":
  621. _print.foreground(TextState.Colors.YELLOW)("rR"),
  622. "new":
  623. _print.foreground(TextState.Colors.LIGHT_GREEN)("N"),
  624. "newslot":
  625. _print.foreground(TextState.Colors.LIGHT_GREEN)("NS"),
  626. "downgrading": (
  627. _print.foreground(TextState.Colors.LIGHT_CYAN)("U") +
  628. _print.foreground(TextState.Colors.LIGHT_BLUE)("D"))}
  629. for pkg in sorted([PackageInformation.add_info(x) for x in
  630. pkglist],
  631. key=lambda y: y['CATEGORY/PN']):
  632. install_flag = ""
  633. if remove_list:
  634. pkgcolor = _print.foreground(remove_color)
  635. else:
  636. for flag in flag_map:
  637. if pkg[flag]:
  638. install_flag = "(%s) " % flag_map[flag]
  639. break
  640. if self.is_binary_pkg(pkg):
  641. pkgcolor = _print.foreground(binary_color)
  642. else:
  643. pkgcolor = _print.foreground(ebuild_color)
  644. if pkg.info['DESCRIPTION']:
  645. fullname = "%s " % _(pkg.info['DESCRIPTION'])
  646. fullname = fullname[:1].upper() + fullname[1:]
  647. else:
  648. fullname = ""
  649. shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
  650. if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB":
  651. size = " (%s)" % pkg["SIZE"]
  652. else:
  653. size = ""
  654. mult = _print.bold("*")
  655. self.printDefault(
  656. _u8("&nbsp;{mult} {fullname}{flag}{shortname}{size}").format(
  657. mult=_u8(mult), fullname=_u8(fullname), shortname=_u8(shortname),
  658. size=_u8(size),
  659. flag=_u8(install_flag)))
  660. def _display_install_package(self, emerge, emergelike=False):
  661. """
  662. Отобразить список устанавливаемых пакетов
  663. """
  664. # подробный список пакетов
  665. _print = self.color_print
  666. if emergelike:
  667. self.printPre(str(emerge.install_packages))
  668. else:
  669. pkglist = emerge.install_packages.list
  670. self.printSUCCESS(_print(
  671. _("Listing packages for installation")))
  672. self._display_pretty_package_list(pkglist)
  673. if emerge.install_packages.remove_list:
  674. self.printSUCCESS(_print(
  675. _("Listing packages for removal")))
  676. self._display_pretty_package_list(
  677. emerge.install_packages.remove_list, remove_list=True)
  678. if len(emerge.install_packages.list) > 0:
  679. install_mess = (_("{count} packages will be installed").format(
  680. count=len(emerge.install_packages.list)) + ", ")
  681. else:
  682. install_mess = ""
  683. if str(emerge.download_size) != "0 kB":
  684. self.printSUCCESS(_("{install}{size} will be downloaded").format(
  685. install=install_mess,
  686. size=str(emerge.download_size)))
  687. def _display_remove_list(self, emerge):
  688. """
  689. Отобразить список удаляемых пакетов
  690. """
  691. # подробный список пакетов
  692. if self.clVars.Get('update.cl_update_emergelist_set') == 'on':
  693. self.printPre(self._emerge_translate(
  694. emerge.uninstall_packages.verbose_result))
  695. else:
  696. _print = self.color_print
  697. pkglist = emerge.uninstall_packages.list
  698. self.printSUCCESS(_print.bold(
  699. _("Listing packages for removal")))
  700. self._display_pretty_package_list(pkglist, remove_list=True)
  701. def mark_schedule(self):
  702. """
  703. Установить отметку о запуске запланированной проверки
  704. """
  705. SystemIni(self.clVars).setVar('system', {'last_check': str(int(time.time()))})
  706. def get_default_emerge_opts(self):
  707. if self.clVars.GetBool('cl_update_with_bdeps_set'):
  708. bdeps = " --with-bdeps=y"
  709. else:
  710. bdeps = " --with-bdeps=n"
  711. return self.clVars.Get('cl_emerge_default_opts') + bdeps
  712. def premerge(self, param, *packages):
  713. """
  714. Вывести информацию об обновлении
  715. """
  716. deo = self.clVars.Get('cl_emerge_default_opts')
  717. param = [param, "-pv"]
  718. with EmergeParser(EmergeCommand(list(packages), emerge_default_opts=deo,
  719. extra_params=param)) as emerge:
  720. try:
  721. emerge.run()
  722. if not emerge.install_packages.list:
  723. self.printSUCCESS(_("The system is up to date"))
  724. self.set_need_update(False)
  725. return True
  726. emergelike = self.clVars.Get('cl_update_emergelist_set') == 'on'
  727. self._display_install_package(emerge, emergelike)
  728. if str(emerge.skipped_packages):
  729. self._display_error(emerge.skipped_packages)
  730. except EmergeError:
  731. self.set_need_update(False)
  732. self._display_install_package(emerge, emergelike=True)
  733. self._display_error(emerge.prepare_error)
  734. raise
  735. if self.clVars.Get('cl_update_pretend_set') == 'on':
  736. # установить кэш: есть обновления
  737. self.set_need_update()
  738. return True
  739. self.set_need_update(False)
  740. answer = self.askConfirm(
  741. _("Would you like to merge these packages?"), "yes")
  742. if answer == "no":
  743. raise KeyboardInterrupt
  744. return "yes"
  745. return True
  746. def set_need_update(self, val=True):
  747. """
  748. Установить флаг: есть обновления
  749. """
  750. if self.clVars.Get('update.cl_update_autocheck_set') == 'off':
  751. val = False
  752. UpdateInfo.set_update_ready(val)
  753. return True
  754. def _emerge_translate(self, s):
  755. """
  756. Перевести текст из emerge
  757. """
  758. return RegexpLocalization('cl_emerge').translate(str(s))
  759. def setUpToDateCache(self):
  760. """
  761. Установить кэш - "нет пакетов для обновления"
  762. """
  763. #self.updateCache(PackageList([]))
  764. self.set_need_update(False)
  765. return True
  766. def _startEmerging(self, emerge):
  767. """
  768. Настроить и выполнить emerge
  769. """
  770. if emerge.install_packages and emerge.install_packages.list:
  771. for pkg in emerge.install_packages.list:
  772. rv = pkg.get('REPLACING_VERSIONS', '')
  773. if rv:
  774. self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \
  775. rv.partition(":")[0]
  776. emerge.command.send("yes\n")
  777. emerge.emerging.add_observer(self._printEmergePackage)
  778. emerge.installing.add_observer(self._printInstallPackage)
  779. emerge.uninstalling.add_observer(self._printUninstallPackage)
  780. emerge.fetching.add_observer(self._printFetching)
  781. def cancel_observing_fetch(fn):
  782. emerge.fetching.clear_observers()
  783. emerge.fetching.add_observer(cancel_observing_fetch)
  784. try:
  785. emerge.run()
  786. except EmergeError:
  787. if emerge.emerging_error:
  788. self._display_error(emerge.emerging_error.log)
  789. else:
  790. self._display_error(emerge.prepare_error)
  791. raise
  792. def _display_error(self, error):
  793. lines_num = int(self.clVars.Get('update.cl_update_lines_limit'))
  794. error = "<br/>".join(str(error).split('<br/>')[-lines_num:])
  795. self.printPre(self._emerge_translate(error))
  796. def emerge(self, use, param, *packages):
  797. """
  798. Выполнить сборку пакета
  799. """
  800. deo = self.clVars.Get('cl_emerge_default_opts')
  801. if not packages:
  802. packages = [param]
  803. extra_params = None
  804. else:
  805. extra_params = [param]
  806. command = EmergeCommand(list(packages), emerge_default_opts=deo,
  807. extra_params=extra_params,
  808. use=use)
  809. if self.clVars.Get('cl_chroot_path') != '/':
  810. command = Chroot(self.clVars.Get('cl_chroot_path'), command)
  811. with EmergeParser(command) as emerge:
  812. try:
  813. emerge.question.action = lambda x: False
  814. emerge.run()
  815. if not emerge.install_packages.list:
  816. return True
  817. except EmergeError:
  818. self._display_error(emerge.prepare_error)
  819. raise
  820. self._startEmerging(emerge)
  821. return True
  822. def emerge_ask(self, pretend, *params):
  823. """
  824. Вывести информацию об обновлении
  825. """
  826. deo = self.get_default_emerge_opts()
  827. param = [x for x in params if x.startswith("-")]
  828. packages = [x for x in params if not x.startswith("-")]
  829. command = EmergeCommand(list(packages), emerge_default_opts=deo,
  830. extra_params=param)
  831. with EmergeParser(command) as emerge:
  832. try:
  833. emerge.question.action = lambda x: False
  834. emerge.run()
  835. if emerge.install_packages.list:
  836. emergelike = self.clVars.Get(
  837. 'update.cl_update_emergelist_set') == 'on'
  838. self._display_install_package(emerge, emergelike)
  839. if emerge.skipped_packages:
  840. self._display_error(emerge.skipped_packages)
  841. if not pretend:
  842. answer = self.askConfirm(
  843. _("Would you like to merge these packages?"), "yes")
  844. if answer == "no":
  845. emerge.command.send("no\n")
  846. raise KeyboardInterrupt
  847. else:
  848. return True
  849. else:
  850. self.set_need_update(False)
  851. self.printSUCCESS(_("The system is up to date"))
  852. except EmergeError:
  853. self.set_need_update(False)
  854. self._display_install_package(emerge, emergelike=True)
  855. self._display_error(emerge.prepare_error)
  856. raise
  857. self._startEmerging(emerge)
  858. return True
  859. def depclean(self):
  860. """
  861. Выполнить очистку системы от лишних пакетов
  862. """
  863. deo = self.get_default_emerge_opts()
  864. emerge = None
  865. try:
  866. emerge = EmergeParser(EmergeCommand(["--depclean"],
  867. emerge_default_opts=deo))
  868. outdated_kernel = False
  869. try:
  870. emerge.question.action = lambda x: False
  871. emerge.run()
  872. if not emerge.uninstall_packages.list:
  873. UpdateInfo(self.clVars).outdated_kernel = False
  874. return True
  875. kernel_pkg = self.clVars.Get('cl_update_kernel_pkg')
  876. if any(("%s-%s" % (x['CATEGORY/PN'], x['PVR'])) == kernel_pkg
  877. for x in emerge.uninstall_packages.list):
  878. pkglist = [
  879. "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in
  880. emerge.uninstall_packages.list
  881. if ("%s-%s" % (x['CATEGORY/PN'],
  882. x['PVR'])) != kernel_pkg]
  883. emerge.command.send('n\n')
  884. emerge.close()
  885. emerge = None
  886. if not pkglist:
  887. UpdateInfo(self.clVars).outdated_kernel = True
  888. return True
  889. emerge = EmergeParser(
  890. EmergeCommand(pkglist,
  891. extra_params=["--unmerge", '--ask=y'],
  892. emerge_default_opts=deo))
  893. emerge.question.action = lambda x: False
  894. emerge.run()
  895. outdated_kernel = True
  896. else:
  897. outdated_kernel = False
  898. self._display_remove_list(emerge)
  899. except EmergeError:
  900. self._display_error(emerge.prepare_error)
  901. raise
  902. if (self.askConfirm(
  903. _("Would you like to unmerge these unused packages "
  904. "(recommended)?")) != 'yes'):
  905. return True
  906. UpdateInfo(self.clVars).outdated_kernel = outdated_kernel
  907. self._startEmerging(emerge)
  908. finally:
  909. if emerge:
  910. emerge.close()
  911. return True
  912. def update_task(self, task_name):
  913. """
  914. Декоратор для добавления меток запуска и останова задачи
  915. """
  916. def decor(f):
  917. def wrapper(*args, **kwargs):
  918. logger = EmergeLog(EmergeLogNamedTask(task_name))
  919. logger.mark_begin_task()
  920. ret = f(*args, **kwargs)
  921. if ret:
  922. logger.mark_end_task()
  923. return ret
  924. return wrapper
  925. return decor
  926. def migrateCacheRepository(self, url, branch, storage):
  927. """
  928. Перенести репозиторий из кэша в локальный
  929. """
  930. rep = storage.get_repository(url, branch)
  931. if rep:
  932. rep.storage = storage.storages[0]
  933. self.clVars.Invalidate('cl_update_profile_storage')
  934. return True
  935. def reconfigureProfileVars(self, profile_dv, chroot):
  936. """
  937. Синхронизировать репозитории
  938. """
  939. dv = self.clVars
  940. try:
  941. if not profile_dv:
  942. raise UpdateError(
  943. _("Failed to use the new profile. Try again."))
  944. for var_name in ('cl_update_rep_path',
  945. 'cl_update_rep_url',
  946. 'cl_update_rep_name',
  947. 'cl_update_branch',
  948. 'cl_update_binhost_list',
  949. 'cl_update_binhost_unstable_list',
  950. 'cl_update_binhost_stable_set',
  951. 'cl_update_binhost_stable_opt_set',
  952. 'cl_update_branch_name',
  953. 'cl_profile_system',
  954. 'cl_update_rep'):
  955. dv.Set(var_name, profile_dv.Get(var_name), force=True)
  956. dv.Set('cl_chroot_path', chroot, force=True)
  957. except DataVarsError as e:
  958. print str(e)
  959. raise UpdateError(_("Wrong profile"))
  960. return True
  961. def setProfile(self, profile_shortname):
  962. profile = self.clVars.Select('cl_update_profile_path',
  963. where='cl_update_profile_shortname',
  964. eq=profile_shortname, limit=1)
  965. if not profile:
  966. raise UpdateError(_("Failed to determine profile %s") %
  967. self.clVars.Get('cl_update_profile_system'))
  968. profile_path = path.relpath(profile, '/etc/portage')
  969. try:
  970. profile_file = '/etc/portage/make.profile'
  971. if not path.exists(
  972. path.join(path.dirname(profile_file), profile_path)):
  973. raise UpdateError(
  974. _("Failed to set the profile: %s") % _("Profile not found"))
  975. for rm_fn in filter(path.lexists,
  976. ('/etc/make.profile',
  977. '/etc/portage/make.profile')):
  978. os.unlink(rm_fn)
  979. os.symlink(profile_path, profile_file)
  980. except (OSError, IOError) as e:
  981. raise UpdateError(_("Failed to set the profile: %s") % str(e))
  982. return True
  983. def applyProfileTemplates(self, useClt=None, cltFilter=False,
  984. useDispatch=True, action="merge"):
  985. """
  986. Наложить шаблоны из профиля
  987. """
  988. from calculate.lib.cl_template import TemplatesError, ProgressTemplate
  989. dv = DataVarsUpdate()
  990. try:
  991. dv.importUpdate()
  992. dv.flIniFile()
  993. dv.Set('cl_action', action, force=True)
  994. dv.Set('cl_templates_locate',
  995. self.clVars.Get('cl_update_templates_locate'))
  996. dv.Set("cl_chroot_path", '/', True)
  997. dv.Set("cl_root_path", '/', True)
  998. for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
  999. "update.cl_update_world"):
  1000. dv.Set(copyvar, self.clVars.Get(copyvar), True)
  1001. # определение каталогов содержащих шаблоны
  1002. useClt = useClt in ("on", True)
  1003. self.addProgress()
  1004. nullProgress = lambda *args, **kw: None
  1005. dispatch = self.dispatchConf if useDispatch else None
  1006. clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt,
  1007. cltFilter=cltFilter,
  1008. printSUCCESS=self.printSUCCESS,
  1009. printWARNING=self.printWARNING,
  1010. askConfirm=self.askConfirm,
  1011. dispatchConf=dispatch,
  1012. printERROR=self.printERROR)
  1013. try:
  1014. clTempl.applyTemplates()
  1015. if clTempl.hasError():
  1016. if clTempl.getError():
  1017. raise TemplatesError(clTempl.getError())
  1018. finally:
  1019. if clTempl:
  1020. if clTempl.cltObj:
  1021. clTempl.cltObj.closeFiles()
  1022. clTempl.closeFiles()
  1023. finally:
  1024. dv.close()
  1025. return True
  1026. def cleanpkg(self):
  1027. """
  1028. Очистить PKGDIR и DISTFILES в текущей системе
  1029. """
  1030. portdirs = ([self.clVars.Get('cl_portdir')] +
  1031. self.clVars.Get('cl_portdir_overlay'))
  1032. pkgfiles = get_packages_files_directory(*portdirs)
  1033. distdirfiles = get_manifest_files_directory(*portdirs)
  1034. distdir = self.clVars.Get('install.cl_distfiles_path')
  1035. pkgdir = self.clVars.Get('cl_pkgdir')
  1036. logger = log("update_cleanpkg.log",
  1037. filename="/var/log/calculate/update_cleanpkg.log",
  1038. formatter="%(asctime)s - %(clean)s - %(message)s")
  1039. return self._cleanpkg(
  1040. distdir, pkgdir, distdirfiles, pkgfiles, logger)
  1041. def _update_binhost_packages(self):
  1042. os.system('/usr/sbin/emaint binhost -f &>/dev/null')
  1043. def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger):
  1044. """
  1045. Общий алгоритм очистки distfiles и pkgdir от устаревших пакетов
  1046. """
  1047. skip_files = ["/metadata.dtd", "/Packages"]
  1048. try:
  1049. if self.clVars.Get('client.os_remote_auth'):
  1050. skip_files += ['portage_lockfile']
  1051. except DataVarsError:
  1052. pass
  1053. for cleantype, filelist in (
  1054. ("packages",
  1055. get_remove_list(pkgdir, list(pkgfiles), depth=4)),
  1056. ("distfiles",
  1057. get_remove_list(distdir, list(distdirfiles), depth=1))):
  1058. removelist = []
  1059. for fn in filelist:
  1060. try:
  1061. if not any(fn.endswith(x) for x in skip_files):
  1062. os.unlink(fn)
  1063. removelist.append(path.basename(fn))
  1064. except OSError:
  1065. pass
  1066. removelist_str = ",".join(removelist)
  1067. if removelist_str:
  1068. logger.info(removelist_str, extra={'clean': cleantype})
  1069. if cleantype == "packages":
  1070. try:
  1071. self._update_binhost_packages()
  1072. for dn in listDirectory(pkgdir, fullPath=True):
  1073. if path.isdir(dn) and not listDirectory(dn):
  1074. os.rmdir(dn)
  1075. except OSError:
  1076. pass
  1077. return True
  1078. def updateSetupCache(self):
  1079. cache = SetupCache(self.clVars)
  1080. cache.update(force=True)
  1081. return True
  1082. def check_binhost(self, write_binhost=True):
  1083. """
  1084. Проверить, что доступен хотя бы один из binhost'ов
  1085. :return:
  1086. """
  1087. hosts = self.clVars.Get("update.cl_update_binhost_host")
  1088. datas = self.clVars.Get("update.cl_update_binhost_revisions")
  1089. bin_cache_fn = pathJoin(self.clVars.Get('cl_chroot_path'),
  1090. LayeredIni.IniPath.Grp)
  1091. if not hosts:
  1092. self.clVars.Delete('cl_update_binhost', location="system")
  1093. try:
  1094. if path.exists(bin_cache_fn):
  1095. os.unlink(bin_cache_fn)
  1096. except OSError:
  1097. raise UpdateError(
  1098. _("Failed to remove cached ini.env of binary repository"))
  1099. raise UpdateError(_("Failed to find the server with "
  1100. "appropriate updates"))
  1101. else:
  1102. with writeFile(bin_cache_fn) as f:
  1103. f.write(datas[0].strip()+"\n")
  1104. if write_binhost:
  1105. if hosts[0] != self.clVars.Get('update.cl_update_binhost'):
  1106. self.refresh_binhost = True
  1107. self.clVars.Set('cl_update_package_cache_set', 'on')
  1108. self.clVars.Write('cl_update_binhost', hosts[0], location="system")
  1109. new_ts = self.clVars.Get("update.cl_update_binhost_timestamp")
  1110. if new_ts:
  1111. new_ts = new_ts[0]
  1112. if new_ts.isdigit():
  1113. ini = SystemIni(self.clVars)
  1114. ini.setVar('system', {'last_update': new_ts})
  1115. if self.clVars.Get('cl_action') == 'sync':
  1116. value = self.clVars.GetBool('cl_update_binhost_stable_set')
  1117. new_value = self.clVars.GetBool('cl_update_binhost_stable_opt_set')
  1118. if value != new_value:
  1119. self.clVars.Write(
  1120. 'cl_update_binhost_stable_set',
  1121. self.clVars.Get('update.cl_update_binhost_stable_opt_set'),
  1122. location="system")
  1123. return True
  1124. def save_with_bdeps(self):
  1125. oldval = self.clVars.Get('cl_update_with_bdeps_set')
  1126. newval = self.clVars.Get('cl_update_with_bdeps_opt_set')
  1127. if oldval != newval:
  1128. self.clVars.Write('cl_update_with_bdeps_set', newval,
  1129. location="system")
  1130. self.clVars.Set('cl_update_force_depclean_set', 'on')
  1131. return True
  1132. def message_binhost_changed(self):
  1133. if self.refresh_binhost:
  1134. self.printWARNING(_("Update server was changed to %s") %
  1135. self.clVars.Get('cl_update_binhost'))
  1136. else:
  1137. self.printSUCCESS(_("Update server %s") %
  1138. self.clVars.Get('cl_update_binhost'))
  1139. return True
  1140. def update_binhost_list(self):
  1141. """
  1142. Обновить список binhost'ов после обновления до master веток
  1143. :return:
  1144. """
  1145. dv = DataVarsUpdate()
  1146. try:
  1147. dv.importUpdate()
  1148. dv.flIniFile()
  1149. changes = False
  1150. for varname in ('update.cl_update_binhost_list',
  1151. 'update.cl_update_binhost_unstable_list',
  1152. 'update.cl_update_binhost_timestamp_path',
  1153. 'cl_update_binhost_revision_path'):
  1154. new_value = dv.Get(varname)
  1155. old_value = self.clVars.Get(varname)
  1156. if new_value != old_value:
  1157. changes = True
  1158. self.clVars.Set(varname, new_value, force=True)
  1159. if not changes:
  1160. self.clVars.Delete('cl_update_binhost', location="system")
  1161. raise UpdateError(_("Failed to find the server with "
  1162. "appropriate updates"))
  1163. except DataVarsError:
  1164. self.clVars.Delete('cl_update_binhost', location="system")
  1165. raise UpdateError(_("Failed to find the server with "
  1166. "appropriate updates"))
  1167. return True
  1168. def drop_binhosts(self, dv):
  1169. """
  1170. Обновление до master веток
  1171. """
  1172. branch = dv.Get('update.cl_update_branch')
  1173. revs = [
  1174. branch for x in dv.Get('update.cl_update_rep_name')
  1175. ]
  1176. dv.Set('update.cl_update_branch_name', revs)
  1177. dv.Invalidate('update.cl_update_rep_rev')
  1178. return True
  1179. def download_packages(self, url_binhost, packages_fn):
  1180. fetch_packages(url_binhost, packages_fn)
  1181. return True