You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1232 lines
49 KiB

  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 Mir Calculate. http://www.calculate-linux.org
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from functools import wraps
  16. import random
  17. import sys
  18. from os import path
  19. import os
  20. import time
  21. from calculate.core.server.gen_pid import search_worked_process
  22. from calculate.core.setup_cache import Cache as SetupCache
  23. from calculate.core.server.func import MethodsInterface
  24. from calculate.lib.cl_template import SystemIni
  25. from calculate.lib.datavars import DataVarsError
  26. from calculate.lib.utils.tools import AddonError
  27. from calculate.lib.utils.colortext.palette import TextState
  28. from calculate.lib.utils.colortext import get_color_print
  29. from calculate.update.emerge_parser import RevdepPercentBlock
  30. from calculate.update.datavars import DataVarsUpdate
  31. from calculate.update.update_info import UpdateInfo
  32. from calculate.lib.cl_log import log
  33. import re
  34. import shutil
  35. from collections import MutableSet
  36. from update_tasks import EmergeMark
  37. from calculate.lib.utils.git import Git, GitError, MTimeKeeper
  38. from calculate.lib.utils.portage import (Layman, EmergeLog,
  39. EmergeLogNamedTask, PackageList,
  40. PackageInformation,
  41. get_packages_files_directory,
  42. get_manifest_files_directory,
  43. get_remove_list)
  44. Colors = TextState.Colors
  45. from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
  46. PercentProgress, process, getRunCommands,
  47. readFile, listDirectory)
  48. import emerge_parser
  49. import logging
  50. from emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
  51. EmergeCache, Chroot)
  52. from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
  53. RegexpLocalization, _)
  54. setLocalTranslate('cl_update3', sys.modules[__name__])
  55. __ = getLazyLocalTranslate(_)
  56. class UpdateError(AddonError):
  57. """Update Error"""
  58. class OverlayOwnCache(MutableSet):
  59. """
  60. Сет оверлеев с интегрированным кэшем
  61. """
  62. def __init__(self, dv=None):
  63. self.dv = dv
  64. def __get_overlays(self):
  65. own_cache_value = SystemIni(self.dv).getVar('update', 'own_cache') or ""
  66. return [x.strip() for x in own_cache_value.split(',') if x.strip()]
  67. def __write_overlays(self, overlays):
  68. if not overlays:
  69. SystemIni(self.dv).delVar('update', 'own_cache')
  70. else:
  71. SystemIni(self.dv).setVar('update',
  72. {'own_cache': ",".join(overlays)})
  73. def __contains__(self, item):
  74. return item in self.__get_overlays()
  75. def __iter__(self):
  76. return iter(self.__get_overlays())
  77. def __len__(self):
  78. return len(self.__get_overlays())
  79. def __append_value(self, overlays, value):
  80. if value not in overlays:
  81. overlays.append(value)
  82. self.__write_overlays(overlays)
  83. def add(self, value):
  84. overlays = self.__get_overlays()
  85. self.__append_value(overlays, value)
  86. def discard(self, value):
  87. overlays = self.__get_overlays()
  88. if value in overlays:
  89. overlays.remove(value)
  90. self.__write_overlays(overlays)
  91. def variable_module(var_env):
  92. def variable_module_decor(f):
  93. @wraps(f)
  94. def wrapper(self, *args, **kw):
  95. old_env = self.clVars.defaultModule
  96. try:
  97. self.clVars.defaultModule = var_env
  98. return f(self, *args, **kw)
  99. finally:
  100. self.clVars.defaultModule = old_env
  101. return wrapper
  102. return variable_module_decor
  103. class Update(MethodsInterface):
  104. """Основной объект для выполнения действий связанных с обновлением системы
  105. """
  106. def init(self):
  107. commandLog = path.join(self.clVars.Get('core.cl_log_path'),
  108. 'lastcommand.log')
  109. emerge_parser.CommandExecutor.logfile = commandLog
  110. self.color_print = get_color_print()
  111. self.emerge_cache = EmergeCache()
  112. if self.clVars.Get('cl_env_debug_set') == 'off':
  113. EmergeCache.logger.logger.setLevel(logging.WARNING)
  114. self.emerge_cache.check_list = (
  115. self.emerge_cache.check_list +
  116. map(emerge_parser.GitCheckvalue,
  117. self.clVars.Get('update.cl_update_rep_path')))
  118. self.update_map = {}
  119. self.refresh_binhost = False
  120. def get_prog_path(self, program_name):
  121. return getProgPath(program_name)
  122. def _syncRepository(self, name, url, rpath, revision,
  123. cb_progress=None, clean=False):
  124. """
  125. Синхронизировать репозитори
  126. """
  127. dv = self.clVars
  128. git = Git()
  129. info_outdated = False
  130. old_dir = "%s.old" % git._gitDir(rpath)
  131. if path.exists(old_dir):
  132. clean = True
  133. try:
  134. self.stash_cache(rpath, name)
  135. if not git.checkExistsRep(rpath):
  136. git.cloneTagRepository(url, rpath, revision,
  137. cb_progress=cb_progress)
  138. info_outdated = True
  139. else:
  140. cr = ""
  141. try:
  142. need_update = False
  143. tag_cr = git.getCommit(rpath, revision)
  144. cr = git.getCurrentCommit(rpath)
  145. ref_type = git.reference_type(rpath, revision)
  146. if git.isNeedUnpack(rpath):
  147. need_update = True
  148. clean = True
  149. elif tag_cr != cr or ref_type == Git.Reference.Branch:
  150. need_update = True
  151. elif clean:
  152. status = git.getStatusInfo(rpath)
  153. if not status or status['files']:
  154. need_update = True
  155. except GitError:
  156. need_update = True
  157. if need_update:
  158. git.updateTagRepository(url, rpath, revision,
  159. cb_progress=cb_progress,
  160. clean=clean)
  161. new_cr = git.getCurrentCommit(rpath)
  162. if new_cr != cr:
  163. info_outdated = True
  164. if info_outdated:
  165. self.raiseOutdate()
  166. dv.Set('cl_update_outdate_set', 'on', force=True)
  167. finally:
  168. self.unstash_cache(rpath, name)
  169. return True
  170. def raiseOutdate(self):
  171. self.clVars.Set('cl_update_outdate_set', 'on', force=True)
  172. def setAutocheckParams(self, status, interval, update_other, cleanpkg):
  173. """
  174. Настроить параметры автопроверки обновлений
  175. """
  176. onoff = lambda x: "on" if x else "off"
  177. self.clVars.Write('cl_update_autocheck_set', onoff(status), True)
  178. self.clVars.Write('cl_update_autocheck_interval', interval, True)
  179. self.clVars.Write('cl_update_other_set', onoff(update_other), True)
  180. self.clVars.Write('cl_update_cleanpkg_set', onoff(cleanpkg), True)
  181. if not status:
  182. UpdateInfo.set_update_ready(False)
  183. return True
  184. def checkSchedule(self, interval, status):
  185. """
  186. Проверить по расписанию необходимость запуска команды
  187. """
  188. if not status:
  189. self.printWARNING(_("Updates autocheck is not enabled"))
  190. return False
  191. last_check = SystemIni().getVar('update', 'last_check') or ""
  192. re_interval = re.compile("^(\d+)\s*(hours?|days?|weeks?)?", re.I)
  193. interval_match = re_interval.search(interval)
  194. MINUTE = 60
  195. HOUR = MINUTE * 60
  196. DAY = HOUR * 24
  197. WEEK = DAY * 7
  198. if interval_match:
  199. if interval_match.group(2):
  200. suffix_map = {'h': HOUR, 'd': DAY, 'w': WEEK}
  201. k = suffix_map.get(interval_match.group(2).lower()[0], HOUR)
  202. else:
  203. k = HOUR
  204. est = int(interval_match.group(1)) * k
  205. else:
  206. est = 3 * HOUR
  207. if last_check:
  208. if last_check.isdigit():
  209. if (time.time() - int(last_check)) < (est - 10 * MINUTE):
  210. self.printWARNING(_("Please wait for the update time"))
  211. return False
  212. self.mark_schedule()
  213. return True
  214. def checkRun(self, wait_update):
  215. """
  216. Проверить повторный запуск
  217. """
  218. update_running = lambda: any(os.getpid() != x
  219. for x in
  220. search_worked_process('update', dv))
  221. dv = self.clVars
  222. if update_running():
  223. if not wait_update:
  224. raise UpdateError(_("Update is already running. "
  225. "Try to run later."))
  226. else:
  227. self.startTask(_("Waiting for another update to be complete"))
  228. while update_running():
  229. self.pauseProcess()
  230. while update_running():
  231. time.sleep(0.3)
  232. self.resumeProcess()
  233. time.sleep(random.random() * 3)
  234. self.endTask()
  235. if self.clVars.Get('cl_chroot_status') == 'off':
  236. emerge_running = lambda: any("/usr/bin/emerge" in x
  237. for x in getRunCommands(True))
  238. if emerge_running():
  239. if not wait_update:
  240. raise UpdateError(_("Emerge is running. "
  241. "Try to run later."))
  242. else:
  243. self.startTask(_("Waiting for emerge to be complete"))
  244. while emerge_running():
  245. time.sleep(1)
  246. self.endTask()
  247. return True
  248. @variable_module("update")
  249. def trimRepositories(self, repname):
  250. """
  251. Синхронизировать репозитории
  252. """
  253. dv = self.clVars
  254. rpath = \
  255. dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1)
  256. git = Git()
  257. self.addProgress()
  258. git.trimRepository(rpath, cb_progress=self.setProgress)
  259. return True
  260. @variable_module("update")
  261. def syncRepositories(self, repname, clean_on_error=True):
  262. """
  263. Синхронизировать репозитории
  264. """
  265. dv = self.clVars
  266. check_status = dv.GetBool('update.cl_update_check_rep_set')
  267. url, rpath, revision = (
  268. dv.Select(["cl_update_rep_url", "cl_update_rep_path",
  269. "cl_update_rep_rev"],
  270. where="cl_update_rep_name", eq=repname, limit=1))
  271. if not url or not rpath:
  272. raise UpdateError(_("Configuration variables for repositories "
  273. "are not setup"))
  274. git = Git()
  275. if not git.checkUrl(url):
  276. raise UpdateError(_("Git %s is unavailable") % url)
  277. chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
  278. if chroot_path == '/':
  279. rpath_orig = rpath
  280. else:
  281. rpath_orig = rpath[len(chroot_path):]
  282. self.addProgress()
  283. mtime = MTimeKeeper(path.join(rpath, "profiles/updates"))
  284. mtime.save()
  285. try:
  286. if clean_on_error:
  287. try:
  288. layman = Layman(dv.Get('cl_update_layman_installed'),
  289. dv.Get('cl_update_layman_make'),
  290. dv.Get('cl_update_layman_conf'),
  291. prefix=chroot_path)
  292. if repname != "portage":
  293. layman.add(repname, url, rpath_orig)
  294. if not self._syncRepository(repname, url, rpath, revision,
  295. cb_progress=self.setProgress,
  296. clean=check_status):
  297. return "skip"
  298. return True
  299. except GitError as e:
  300. if e.addon:
  301. self.printWARNING(str(e.addon))
  302. self.printWARNING(str(e))
  303. self.endTask(False)
  304. self.startTask(
  305. _("Re-fetching the {name} repository").format(
  306. name=repname))
  307. self.addProgress()
  308. rpath_new = "%s_new" % rpath
  309. try:
  310. self._syncRepository(repname, url, rpath_new, revision,
  311. cb_progress=self.setProgress,
  312. clean=check_status)
  313. removeDir(rpath)
  314. shutil.move(rpath_new, rpath)
  315. except OSError as e:
  316. raise UpdateError(_("Failed to modify the "
  317. "{repname} repository").format(
  318. repname=repname) + _(": ") + str(e))
  319. finally:
  320. if path.exists(rpath_new):
  321. removeDir(rpath_new)
  322. else:
  323. if not self._syncRepository(repname, url, rpath, revision,
  324. clean=check_status):
  325. return "skip"
  326. layman = Layman(dv.Get('cl_update_layman_installed'),
  327. dv.Get('cl_update_layman_make'),
  328. dv.Get('cl_update_layman_conf'),
  329. prefix=chroot_path)
  330. if repname != "portage":
  331. layman.add(repname, url, rpath_orig)
  332. finally:
  333. mtime.restore()
  334. return True
  335. metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
  336. def stash_cache(self, rpath, name):
  337. """
  338. Спрятать кэш
  339. """
  340. if name in ("portage",):
  341. return
  342. if not name in OverlayOwnCache(self.clVars):
  343. for cachename in self.metadata_cache_names:
  344. cachedir = path.join(rpath, cachename)
  345. if path.exists(cachedir):
  346. try:
  347. cachedir_s = path.join(path.dirname(rpath),
  348. path.basename(
  349. cachename) + ".stash")
  350. if path.exists(cachedir_s):
  351. removeDir(cachedir_s)
  352. shutil.move(cachedir, cachedir_s)
  353. except BaseException as e:
  354. pass
  355. def unstash_cache(self, rpath, name):
  356. """
  357. Извлеч кэш
  358. """
  359. if name in ("portage",):
  360. return
  361. cachenames = self.metadata_cache_names
  362. if not name in OverlayOwnCache(self.clVars):
  363. if any(path.exists(path.join(rpath, x)) for x in cachenames):
  364. for cachename in cachenames:
  365. cachedir_s = path.join(path.dirname(rpath),
  366. path.basename(cachename) + ".stash")
  367. if path.exists(cachedir_s):
  368. try:
  369. removeDir(cachedir_s)
  370. except BaseException as e:
  371. pass
  372. OverlayOwnCache(self.clVars).add(name)
  373. else:
  374. for cachename in cachenames:
  375. cachedir = path.join(rpath, cachename)
  376. cachedir_s = path.join(path.dirname(rpath),
  377. path.basename(cachename) + ".stash")
  378. if path.exists(cachedir_s):
  379. try:
  380. shutil.move(cachedir_s, cachedir)
  381. except BaseException as e:
  382. pass
  383. else:
  384. if all(not path.exists(path.join(rpath, x)) for x in cachenames):
  385. OverlayOwnCache(self.clVars).discard(name)
  386. def syncLaymanRepository(self, repname):
  387. """
  388. Обновить репозиторий через layman
  389. """
  390. layman = self.get_prog_path('/usr/bin/layman')
  391. if not layman:
  392. raise UpdateError(_("The Layman tool is not found"))
  393. rpath = self.clVars.Select('cl_update_other_rep_path',
  394. where='cl_update_other_rep_name', eq=repname,
  395. limit=1)
  396. laymanname = path.basename(rpath)
  397. self.stash_cache(rpath, laymanname)
  398. try:
  399. if Git.is_git(rpath):
  400. self.addProgress()
  401. p = PercentProgress(layman, "-s", laymanname, part=1, atty=True)
  402. for perc in p.progress():
  403. self.setProgress(perc)
  404. else:
  405. p = process(layman, "-s", repname, stderr=STDOUT)
  406. if p.failed():
  407. raise UpdateError(
  408. _("Failed to update the {rname} repository").format(
  409. rname=repname),
  410. addon=p.read())
  411. finally:
  412. self.unstash_cache(rpath, laymanname)
  413. return True
  414. def _regenCache_process(self, progname, repname, cpu_num):
  415. return process(progname, "--repo=%s" % repname, "--update",
  416. "--jobs=%s" % cpu_num, stderr=STDOUT)
  417. def regenCache(self, repname):
  418. """
  419. Обновить кэш метаданных репозитория
  420. """
  421. egenCache = self.get_prog_path('/usr/bin/egencache')
  422. if not egenCache:
  423. raise UpdateError(_("The Portage tool is not found"))
  424. if repname in self.clVars.Get('cl_update_rep_name'):
  425. path_rep = self.clVars.Select('cl_update_rep_path',
  426. where='cl_update_rep_name',
  427. eq=repname, limit=1)
  428. repo_name = readFile(
  429. path.join(path_rep, "profiles/repo_name")).strip()
  430. if repo_name != repname:
  431. self.printWARNING(
  432. _("Repository '{repo_name}' called '{repname}'"
  433. " in cl_update_rep_name").format(
  434. repo_name=repo_name, repname=repname))
  435. raise UpdateError(_("Failed to update the cache of the {rname} "
  436. "repository").format(rname=repname))
  437. cpu_num = self.clVars.Get('hr_cpu_num')
  438. if repname in OverlayOwnCache(self.clVars):
  439. self.printWARNING(
  440. _("Repository %s has its own cache") % repname.capitalize())
  441. else:
  442. self.startTask(_("Updating the %s repository cache") %
  443. repname.capitalize())
  444. p = self._regenCache_process(egenCache, repname, cpu_num)
  445. if p.failed():
  446. raise UpdateError(_("Failed to update the cache of the {rname} "
  447. "repository").format(rname=repname),
  448. addon=p.read())
  449. return True
  450. def emergeMetadata(self):
  451. """
  452. Выполнить egencache и emerge --metadata
  453. """
  454. emerge = self.get_prog_path("/usr/bin/emerge")
  455. if not emerge:
  456. raise UpdateError(_("The Emerge tool is not found"))
  457. self.addProgress()
  458. p = PercentProgress(emerge, "--ask=n", "--metadata", part=1, atty=True)
  459. for perc in p.progress():
  460. self.setProgress(perc)
  461. if p.failed():
  462. data = p.read()
  463. with open('/var/log/calculate/failed-metadata-%d.log' % time.time(),
  464. 'w') as f:
  465. f.write(data + p.alldata)
  466. raise UpdateError(_("Failed to update metadata"), addon=data)
  467. return True
  468. def _eixUpdateCommand(self, eix_cmd, countRep):
  469. return PercentProgress(eix_cmd, "-F", part=countRep or 1, atty=True)
  470. def eixUpdate(self, repositroies):
  471. """
  472. Выполенине eix-update для репозиторием
  473. eix-update выполнятется только для тех репозиториев, которые
  474. обновлялись, если cl_update_eixsync_force==auto, либо
  475. все, если cl_update_eixupdate_force==force
  476. """
  477. eixupdate = self.get_prog_path("/usr/bin/eix-update")
  478. if not eixupdate:
  479. raise UpdateError(_("The Eix tool is not found"))
  480. self.addProgress()
  481. countRep = len(repositroies)
  482. p = self._eixUpdateCommand(eixupdate, countRep)
  483. for perc in p.progress():
  484. self.setProgress(perc)
  485. if p.failed():
  486. raise UpdateError(_("Failed to update eix cache"), addon=p.read())
  487. return True
  488. def is_binary_pkg(self, pkg, binary=None):
  489. """
  490. Является ли пакет бинарным
  491. """
  492. if binary:
  493. return True
  494. if 'PN' in pkg and pkg['PN'].endswith('-bin'):
  495. return True
  496. if binary is not None:
  497. return binary
  498. if "binary" in pkg and pkg['binary']:
  499. return True
  500. return False
  501. def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
  502. """
  503. Вывод сообщения сборки пакета
  504. """
  505. self.endTask()
  506. _print = self.color_print
  507. one = _print("{0}", num)
  508. two = _print("{0}", max_num)
  509. part = _("({current} of {maximum})").format(current=one, maximum=two)
  510. _print = _print.foreground(Colors.DEFAULT)
  511. if self.is_binary_pkg(pkg, binary):
  512. _colorprint = _print.foreground(Colors.PURPLE)
  513. else:
  514. _colorprint = _print.foreground(Colors.GREEN)
  515. PackageInformation.add_info(pkg)
  516. name = ""
  517. if pkg.info['DESCRIPTION']:
  518. name = _(pkg.info['DESCRIPTION'])
  519. name = name[:1].upper() + name[1:]
  520. if not name:
  521. name = str(pkg)
  522. self.printSUCCESS(
  523. _("{part} {package}").format(part=part, package=name))
  524. self.startTask(
  525. _("Emerging {package}").format(package=_colorprint(str(pkg))))
  526. def _printInstallPackage(self, pkg, binary=False):
  527. """
  528. Вывод сообщения установки пакета
  529. """
  530. self.endTask()
  531. _print = self.color_print
  532. if self.is_binary_pkg(pkg, binary):
  533. _print = _print.foreground(Colors.PURPLE)
  534. else:
  535. _print = _print.foreground(Colors.GREEN)
  536. pkg_key = "{CATEGORY}/{PF}".format(**pkg)
  537. if pkg_key in self.update_map:
  538. self.startTask(_("Installing {pkg} [{oldver}]").format(
  539. pkg=_print(str(pkg)), oldver=self.update_map[pkg_key]))
  540. self.update_map.pop(pkg_key)
  541. else:
  542. self.startTask(_("Installing %s") % (_print(str(pkg))))
  543. def _printFetching(self, fn):
  544. """
  545. Вывод сообщения о скачивании
  546. """
  547. self.endTask()
  548. self.startTask(_("Fetching binary packages"))
  549. def _printUninstallPackage(self, pkg, num=1, max_num=1):
  550. """
  551. Вывод сообщения удаления пакета
  552. """
  553. self.endTask()
  554. _print = self.color_print
  555. if num and max_num:
  556. one = _print("{0}", num)
  557. two = _print("{0}", max_num)
  558. part = _(" ({current} of {maximum})").format(current=one,
  559. maximum=two)
  560. else:
  561. part = ""
  562. _print = _print.foreground(Colors.LIGHT_RED)
  563. self.startTask(
  564. _("Unmerging{part} {package}").format(part=part,
  565. package=_print(str(pkg))))
  566. def emergelike(self, cmd, *params):
  567. """
  568. Запуск команды, которая подразумевает выполнение emerge
  569. """
  570. cmd_path = self.get_prog_path(cmd)
  571. if not cmd_path:
  572. raise UpdateError(_("Failed to find the %s command") % cmd)
  573. with EmergeParser(
  574. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  575. self._startEmerging(emerge)
  576. return True
  577. def revdep_rebuild(self, cmd, *params):
  578. """
  579. Запуск revdep-rebulid
  580. """
  581. cmd_path = self.get_prog_path(cmd)
  582. if not cmd_path:
  583. raise UpdateError(_("Failed to find the %s command") % cmd)
  584. with EmergeParser(
  585. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  586. revdep = RevdepPercentBlock(emerge)
  587. self.addProgress()
  588. revdep.add_observer(self.setProgress)
  589. revdep.action = lambda x: (
  590. self.endTask(), self.startTask(_("Assigning files to packages"))
  591. if "Assign" in revdep else None)
  592. self._startEmerging(emerge)
  593. return True
  594. def _display_pretty_package_list(self, pkglist, remove_list=False):
  595. """
  596. Отобразить список пакетов в "удобочитаемом" виде
  597. """
  598. _print = self.color_print
  599. ebuild_color = TextState.Colors.GREEN
  600. binary_color = TextState.Colors.PURPLE
  601. remove_color = TextState.Colors.LIGHT_RED
  602. flag_map = {"updating":
  603. _print.foreground(TextState.Colors.LIGHT_CYAN)("U"),
  604. "reinstall":
  605. _print.foreground(TextState.Colors.YELLOW)("rR"),
  606. "new":
  607. _print.foreground(TextState.Colors.LIGHT_GREEN)("N"),
  608. "newslot":
  609. _print.foreground(TextState.Colors.LIGHT_GREEN)("NS"),
  610. "downgrading": (
  611. _print.foreground(TextState.Colors.LIGHT_CYAN)("U") +
  612. _print.foreground(TextState.Colors.LIGHT_BLUE)("D"))}
  613. for pkg in sorted([PackageInformation.add_info(x) for x in
  614. pkglist],
  615. key=lambda y: y['CATEGORY/PN']):
  616. install_flag = ""
  617. if remove_list:
  618. pkgcolor = _print.foreground(remove_color)
  619. else:
  620. for flag in flag_map:
  621. if pkg[flag]:
  622. install_flag = "(%s) " % flag_map[flag]
  623. break
  624. if self.is_binary_pkg(pkg):
  625. pkgcolor = _print.foreground(binary_color)
  626. else:
  627. pkgcolor = _print.foreground(ebuild_color)
  628. if pkg.info['DESCRIPTION']:
  629. fullname = "%s " % _(pkg.info['DESCRIPTION'])
  630. fullname = fullname[:1].upper() + fullname[1:]
  631. else:
  632. fullname = ""
  633. shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
  634. if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB":
  635. size = " (%s)" % pkg["SIZE"]
  636. else:
  637. size = ""
  638. mult = _print.bold("*")
  639. self.printDefault(
  640. "&nbsp;{mult} {fullname}{flag}{shortname}{size}".format(
  641. mult=mult, fullname=fullname, shortname=shortname,
  642. size=size,
  643. flag=install_flag))
  644. def _display_install_package(self, emerge, emergelike=False):
  645. """
  646. Отобразить список устанавливаемых пакетов
  647. """
  648. # подробный список пакетов
  649. _print = self.color_print
  650. if emergelike:
  651. self.printPre(str(emerge.install_packages))
  652. else:
  653. pkglist = emerge.install_packages.list
  654. self.printSUCCESS(_print(
  655. _("Listing packages for installation")))
  656. self._display_pretty_package_list(pkglist)
  657. if emerge.install_packages.remove_list:
  658. self.printSUCCESS(_print(
  659. _("Listing packages for removal")))
  660. self._display_pretty_package_list(
  661. emerge.install_packages.remove_list, remove_list=True)
  662. if len(emerge.install_packages.list) > 0:
  663. install_mess = (_("{count} packages will be installed").format(
  664. count=len(emerge.install_packages.list)) + ", ")
  665. else:
  666. install_mess = ""
  667. if str(emerge.download_size) != "0 kB":
  668. self.printSUCCESS(_("{install}{size} will be downloaded").format(
  669. install=install_mess,
  670. size=str(emerge.download_size)))
  671. def _display_remove_list(self, emerge):
  672. """
  673. Отобразить список удаляемых пакетов
  674. """
  675. # подробный список пакетов
  676. if self.clVars.Get('update.cl_update_emergelist_set') == 'on':
  677. self.printPre(self._emerge_translate(
  678. emerge.uninstall_packages.verbose_result))
  679. else:
  680. _print = self.color_print
  681. pkglist = emerge.uninstall_packages.list
  682. self.printSUCCESS(_print.bold(
  683. _("Listing packages for removal")))
  684. self._display_pretty_package_list(pkglist, remove_list=True)
  685. def getCacheOnWorld(self, params, packages, check=False):
  686. """
  687. Получить список обновляемых пакетов @world из кэша
  688. """
  689. if "@world" in packages:
  690. elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge))
  691. if check and (elog.list or elog.remove_list):
  692. self.emerge_cache.drop_cache(
  693. "Some packages was installed or removed")
  694. return params, packages
  695. installed_pkgs = elog.list
  696. new_packages = self.emerge_cache.get_cached_package_list()
  697. if new_packages is not None:
  698. return "-1O", ["=%s" % x for x in new_packages
  699. if not str(x) in installed_pkgs]
  700. return params, packages
  701. def updateCache(self, pkg_list):
  702. """
  703. Обновить кэш. Оставить отметку в emerge.log о том, выполнено действие
  704. premerge
  705. """
  706. self.emerge_cache.set_cache(pkg_list)
  707. elog = EmergeLog(EmergeLogNamedTask(EmergeMark.Premerge))
  708. elog.mark_end_task(),
  709. def mark_schedule(self):
  710. """
  711. Установить отметку о запуске запланированной проверки
  712. """
  713. SystemIni().setVar('update', {'last_check': str(int(time.time()))})
  714. def get_default_emerge_opts(self):
  715. return self.clVars.Get('cl_emerge_default_opts')
  716. def premerge(self, param, *packages):
  717. """
  718. Вывести информацию об обновлении
  719. """
  720. deo = self.clVars.Get('cl_emerge_default_opts')
  721. param, packages = self.getCacheOnWorld(param, packages, check=True)
  722. param = [param, "-pv"]
  723. if not packages:
  724. self.printSUCCESS(_("Installed packages are up to date"))
  725. self.set_need_update(False)
  726. return True
  727. with EmergeParser(EmergeCommand(list(packages), emerge_default_opts=deo,
  728. extra_params=param)) as emerge:
  729. try:
  730. emerge.run()
  731. if "@world" in packages:
  732. if emerge.install_packages.remove_list:
  733. self.emerge_cache.drop_cache(
  734. "List has packages for remove")
  735. elif emerge.install_packages.block_packages:
  736. self.emerge_cache.drop_cache(
  737. "List has block packages")
  738. else:
  739. self.updateCache(emerge.install_packages.list)
  740. if not emerge.install_packages.list:
  741. self.printSUCCESS(_("The system is up to date"))
  742. self.set_need_update(False)
  743. return True
  744. emergelike = self.clVars.Get('cl_update_emergelist_set') == 'on'
  745. self._display_install_package(emerge, emergelike)
  746. if str(emerge.skipped_packages):
  747. self._display_error(emerge.skipped_packages)
  748. except EmergeError:
  749. self.set_need_update(False)
  750. self.emerge_cache.drop_cache("Emerge error")
  751. self._display_install_package(emerge, emergelike=True)
  752. self._display_error(emerge.prepare_error)
  753. raise
  754. if self.clVars.Get('cl_update_pretend_set') == 'on':
  755. # установить кэш: есть обновления
  756. self.set_need_update()
  757. return True
  758. self.set_need_update(False)
  759. answer = self.askConfirm(
  760. _("Would you like to merge these packages?"), "yes")
  761. if answer == "no":
  762. raise KeyboardInterrupt
  763. return "yes"
  764. return True
  765. def set_need_update(self, val=True):
  766. """
  767. Установить флаг: есть обновления
  768. """
  769. if self.clVars.Get('update.cl_update_autocheck_set') == 'off':
  770. val = False
  771. UpdateInfo.set_update_ready(val)
  772. return True
  773. def _emerge_translate(self, s):
  774. """
  775. Перевести текст из emerge
  776. """
  777. return RegexpLocalization('cl_emerge').translate(str(s))
  778. def setUpToDateCache(self):
  779. """
  780. Установить кэш - "нет пакетов для обновления"
  781. """
  782. self.updateCache(PackageList([]))
  783. return True
  784. def _startEmerging(self, emerge):
  785. """
  786. Настроить и выполнить emerge
  787. """
  788. if emerge.install_packages and emerge.install_packages.list:
  789. for pkg in emerge.install_packages.list:
  790. rv = pkg.get('REPLACING_VERSIONS', '')
  791. if rv:
  792. self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \
  793. rv.partition(":")[0]
  794. emerge.command.send("yes\n")
  795. emerge.emerging.add_observer(self._printEmergePackage)
  796. emerge.installing.add_observer(self._printInstallPackage)
  797. emerge.uninstalling.add_observer(self._printUninstallPackage)
  798. emerge.fetching.add_observer(self._printFetching)
  799. def cancel_observing_fetch(fn):
  800. emerge.fetching.clear_observers()
  801. emerge.fetching.add_observer(cancel_observing_fetch)
  802. try:
  803. emerge.run()
  804. except EmergeError:
  805. self.emerge_cache.drop_cache("Emerge error")
  806. if emerge.emerging_error:
  807. self._display_error(emerge.emerging_error.log)
  808. else:
  809. self._display_error(emerge.prepare_error)
  810. raise
  811. def _display_error(self, error):
  812. lines_num = int(self.clVars.Get('update.cl_update_lines_limit'))
  813. error = "<br/>".join(str(error).split('<br/>')[-lines_num:])
  814. self.printPre(self._emerge_translate(error))
  815. def emerge(self, use, param, *packages):
  816. """
  817. Выполнить сборку пакета
  818. """
  819. deo = self.clVars.Get('cl_emerge_default_opts')
  820. if not packages:
  821. packages = [param]
  822. extra_params = None
  823. else:
  824. param, packages = self.getCacheOnWorld(param, packages)
  825. if not packages:
  826. return True
  827. extra_params = [param]
  828. command = EmergeCommand(list(packages), emerge_default_opts=deo,
  829. extra_params=extra_params,
  830. use=use)
  831. if self.clVars.Get('cl_chroot_path') != '/':
  832. command = Chroot(self.clVars.Get('cl_chroot_path'), command)
  833. with EmergeParser(command) as emerge:
  834. try:
  835. emerge.question.action = lambda x: False
  836. emerge.run()
  837. if not emerge.install_packages.list:
  838. return True
  839. except EmergeError:
  840. self.emerge_cache.drop_cache("Emerge error")
  841. self._display_error(emerge.prepare_error)
  842. raise
  843. self._startEmerging(emerge)
  844. return True
  845. def depclean(self):
  846. """
  847. Выполнить очистку системы от лишних пакетов
  848. """
  849. deo = self.get_default_emerge_opts()
  850. emerge = None
  851. try:
  852. emerge = EmergeParser(EmergeCommand(["--depclean"],
  853. emerge_default_opts=deo))
  854. outdated_kernel = False
  855. try:
  856. emerge.question.action = lambda x: False
  857. emerge.run()
  858. if not emerge.uninstall_packages.list:
  859. UpdateInfo(self.clVars).outdated_kernel = False
  860. return True
  861. kernel_pkg = self.clVars.Get('cl_update_kernel_pkg')
  862. if any(("%s-%s" % (x['CATEGORY/PN'], x['PVR'])) == kernel_pkg
  863. for x in emerge.uninstall_packages.list):
  864. pkglist = [
  865. "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in
  866. emerge.uninstall_packages.list
  867. if ("%s-%s" % (x['CATEGORY/PN'],
  868. x['PVR'])) != kernel_pkg]
  869. emerge.command.send('n\n')
  870. emerge.close()
  871. emerge = None
  872. if not pkglist:
  873. UpdateInfo(self.clVars).outdated_kernel = True
  874. return True
  875. emerge = EmergeParser(
  876. EmergeCommand(pkglist,
  877. extra_params=["--unmerge", '--ask=y'],
  878. emerge_default_opts=deo))
  879. emerge.question.action = lambda x: False
  880. emerge.run()
  881. outdated_kernel = True
  882. else:
  883. outdated_kernel = False
  884. self._display_remove_list(emerge)
  885. except EmergeError:
  886. self._display_error(emerge.prepare_error)
  887. raise
  888. if (self.askConfirm(
  889. _("Would you like to unmerge these unused packages "
  890. "(recommended)?")) != 'yes'):
  891. return True
  892. UpdateInfo(self.clVars).outdated_kernel = outdated_kernel
  893. self._startEmerging(emerge)
  894. finally:
  895. if emerge:
  896. emerge.close()
  897. return True
  898. def update_task(self, task_name):
  899. """
  900. Декоратор для добавления меток запуска и останова задачи
  901. """
  902. def decor(f):
  903. def wrapper(*args, **kwargs):
  904. logger = EmergeLog(EmergeLogNamedTask(task_name))
  905. logger.mark_begin_task()
  906. ret = f(*args, **kwargs)
  907. if ret:
  908. logger.mark_end_task()
  909. return ret
  910. return wrapper
  911. return decor
  912. def migrateCacheRepository(self, url, branch, storage):
  913. """
  914. Перенести репозиторий из кэша в локальный
  915. """
  916. rep = storage.get_repository(url, branch)
  917. if rep:
  918. rep.storage = storage.storages[0]
  919. self.clVars.Invalidate('cl_update_profile_storage')
  920. return True
  921. def reconfigureProfileVars(self, profile_dv, chroot):
  922. """
  923. Синхронизировать репозитории
  924. """
  925. dv = self.clVars
  926. try:
  927. if not profile_dv:
  928. raise UpdateError(
  929. _("Failed to use the new profile. Try again."))
  930. for var_name in ('cl_update_rep_path',
  931. 'cl_update_rep_url',
  932. 'cl_update_rep_name',
  933. 'cl_update_branch',
  934. 'cl_update_binhost_list',
  935. 'cl_update_branch_name',
  936. 'cl_profile_system',
  937. 'cl_update_rep'):
  938. dv.Set(var_name, profile_dv.Get(var_name), force=True)
  939. dv.Set('cl_chroot_path', chroot, force=True)
  940. except DataVarsError as e:
  941. print str(e)
  942. raise UpdateError(_("Wrong profile"))
  943. return True
  944. def setProfile(self, profile_shortname):
  945. profile = self.clVars.Select('cl_update_profile_path',
  946. where='cl_update_profile_shortname',
  947. eq=profile_shortname, limit=1)
  948. if not profile:
  949. raise UpdateError(_("Failed to determine profile %s") %
  950. self.clVars.Get('cl_update_profile_system'))
  951. profile_path = path.relpath(profile, '/etc/portage')
  952. try:
  953. profile_file = '/etc/portage/make.profile'
  954. if not path.exists(
  955. path.join(path.dirname(profile_file), profile_path)):
  956. raise UpdateError(
  957. _("Failed to set the profile: %s") % _("Profile not found"))
  958. for rm_fn in filter(path.lexists,
  959. ('/etc/make.profile',
  960. '/etc/portage/make.profile')):
  961. os.unlink(rm_fn)
  962. os.symlink(profile_path, profile_file)
  963. except (OSError, IOError) as e:
  964. raise UpdateError(_("Failed to set the profile: %s") % str(e))
  965. return True
  966. def applyProfileTemplates(self, useClt=None, cltFilter=False,
  967. useDispatch=True, action="merge"):
  968. """
  969. Наложить шаблоны из профиля
  970. """
  971. from calculate.lib.cl_template import TemplatesError, ProgressTemplate
  972. dv = DataVarsUpdate()
  973. try:
  974. dv.importUpdate()
  975. dv.flIniFile()
  976. dv.Set('cl_action', action, force=True)
  977. dv.Set('cl_templates_locate',
  978. self.clVars.Get('cl_update_templates_locate'))
  979. dv.Set("cl_chroot_path", '/', True)
  980. dv.Set("cl_root_path", '/', True)
  981. for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
  982. "update.cl_update_world"):
  983. dv.Set(copyvar, self.clVars.Get(copyvar), True)
  984. # определение каталогов содержащих шаблоны
  985. useClt = useClt in ("on", True)
  986. self.addProgress()
  987. nullProgress = lambda *args, **kw: None
  988. dispatch = self.dispatchConf if useDispatch else None
  989. clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt,
  990. cltFilter=cltFilter,
  991. printSUCCESS=self.printSUCCESS,
  992. printWARNING=self.printWARNING,
  993. askConfirm=self.askConfirm,
  994. dispatchConf=dispatch,
  995. printERROR=self.printERROR)
  996. try:
  997. clTempl.applyTemplates()
  998. if clTempl.hasError():
  999. if clTempl.getError():
  1000. raise TemplatesError(clTempl.getError())
  1001. finally:
  1002. if clTempl:
  1003. if clTempl.cltObj:
  1004. clTempl.cltObj.closeFiles()
  1005. clTempl.closeFiles()
  1006. finally:
  1007. dv.close()
  1008. return True
  1009. def cleanpkg(self):
  1010. """
  1011. Очистить PKGDIR и DISTFILES в текущей системе
  1012. """
  1013. portdirs = ([self.clVars.Get('cl_portdir')] +
  1014. self.clVars.Get('cl_portdir_overlay'))
  1015. pkgfiles = get_packages_files_directory(*portdirs)
  1016. distdirfiles = get_manifest_files_directory(*portdirs)
  1017. distdir = self.clVars.Get('install.cl_distfiles_path')
  1018. pkgdir = self.clVars.Get('cl_pkgdir')
  1019. logger = log("update_cleanpkg.log",
  1020. filename="/var/log/calculate/update_cleanpkg.log",
  1021. formatter="%(asctime)s - %(clean)s - %(message)s")
  1022. return self._cleanpkg(
  1023. distdir, pkgdir, distdirfiles, pkgfiles, logger)
  1024. def _update_binhost_packages(self):
  1025. os.system('/usr/sbin/emaint binhost -f &>/dev/null')
  1026. def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger):
  1027. """
  1028. Общий алгоритм очистки distfiles и pkgdir от устаревших пакетов
  1029. """
  1030. skip_files = ["/metadata.dtd", "/Packages"]
  1031. try:
  1032. if self.clVars.Get('client.os_remote_auth'):
  1033. skip_files += ['portage_lockfile']
  1034. except DataVarsError:
  1035. pass
  1036. for cleantype, filelist in (
  1037. ("packages",
  1038. get_remove_list(pkgdir, list(pkgfiles), depth=4)),
  1039. ("distfiles",
  1040. get_remove_list(distdir, list(distdirfiles), depth=1))):
  1041. removelist = []
  1042. for fn in filelist:
  1043. try:
  1044. if not any(fn.endswith(x) for x in skip_files):
  1045. os.unlink(fn)
  1046. removelist.append(path.basename(fn))
  1047. except OSError:
  1048. pass
  1049. removelist_str = ",".join(removelist)
  1050. if removelist_str:
  1051. logger.info(removelist_str, extra={'clean': cleantype})
  1052. if cleantype == "packages":
  1053. try:
  1054. self._update_binhost_packages()
  1055. for dn in listDirectory(pkgdir, fullPath=True):
  1056. if path.isdir(dn) and not listDirectory(dn):
  1057. os.rmdir(dn)
  1058. except OSError:
  1059. pass
  1060. return True
  1061. def updateSetupCache(self):
  1062. cache = SetupCache(self.clVars)
  1063. cache.update(force=True)
  1064. return True
  1065. def check_binhost(self, write_binhost=True):
  1066. """
  1067. Проверить, что доступен хотя бы один из binhost'ов
  1068. :return:
  1069. """
  1070. hosts = self.clVars.Get("update.cl_update_binhost_host")
  1071. if not hosts:
  1072. self.clVars.Delete('cl_update_binhost', location="system")
  1073. raise UpdateError(_("Update server is unavailable"))
  1074. if write_binhost:
  1075. if hosts[0] != self.clVars.Get('update.cl_update_binhost'):
  1076. self.refresh_binhost = True
  1077. self.clVars.Write('cl_update_binhost', hosts[0], location="system")
  1078. new_ts = self.clVars.Get("update.cl_update_binhost_timestamp")
  1079. if new_ts:
  1080. new_ts = new_ts[0]
  1081. old_ts = self.clVars.Get("update.cl_update_last_timestamp")
  1082. if new_ts.isdigit() and (not old_ts.isdigit() or
  1083. int(new_ts) > int(old_ts)):
  1084. ini = SystemIni(self.clVars)
  1085. ini.setVar('update', {'last_update': new_ts})
  1086. return True
  1087. def message_binhost_changed(self):
  1088. if self.refresh_binhost:
  1089. self.printWARNING(_("Update server was changed to %s") %
  1090. self.clVars.Get('cl_update_binhost'))
  1091. elif self.clVars.GetBool('update.cl_update_binhost_recheck_set'):
  1092. self.printSUCCESS(_("Update server %s") %
  1093. self.clVars.Get('cl_update_binhost'))
  1094. return True
  1095. def update_binhost_list(self):
  1096. """
  1097. Обновить список binhost'ов после обновления до master веток
  1098. :return:
  1099. """
  1100. dv = DataVarsUpdate()
  1101. try:
  1102. dv.importUpdate()
  1103. dv.flIniFile()
  1104. changes = False
  1105. for varname in ('update.cl_update_binhost_list',
  1106. 'update.cl_update_binhost_timestamp_path',
  1107. 'cl_update_binhost_revision_path'):
  1108. new_value = dv.Get(varname)
  1109. old_value = self.clVars.Get(varname)
  1110. if new_value != old_value:
  1111. changes = True
  1112. self.clVars.Set(varname, new_value, force=True)
  1113. if not changes:
  1114. raise UpdateError(_("Update server is unavailable"))
  1115. except DataVarsError:
  1116. raise UpdateError(_("Update server is unavailable"))
  1117. return True
  1118. def drop_binhosts(self, dv):
  1119. """
  1120. Обновление до master веток
  1121. """
  1122. branch = dv.Get('update.cl_update_branch')
  1123. revs = [
  1124. branch for x in dv.Get('update.cl_update_rep_name')
  1125. ]
  1126. dv.Set('update.cl_update_branch_name', revs)
  1127. dv.Invalidate('update.cl_update_rep_rev')
  1128. return True