No puede seleccionar más de 25 temas Los temas deben comenzar con una letra o número, pueden incluir guiones ('-') y pueden tener hasta 35 caracteres de largo.

update.py 71 KiB

hace 5 años
hace 5 años
hace 5 años
hace 5 años
hace 5 años
hace 5 años
hace 5 años
hace 5 años
hace 5 años
hace 6 años
hace 6 años
hace 7 años
hace 7 años
hace 7 años
hace 7 años
hace 7 años
hace 5 años
hace 5 años
hace 7 años
hace 7 años
hace 7 años
hace 7 años
hace 5 años
hace 5 años
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736
  1. # -*- coding: utf-8 -*-
  2. # Copyright 2014-2016 Mir Calculate. http://www.calculate-linux.org
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License");
  5. # you may not use this file except in compliance with the License.
  6. # You may obtain a copy of the License at
  7. #
  8. # http://www.apache.org/licenses/LICENSE-2.0
  9. #
  10. # Unless required by applicable law or agreed to in writing, software
  11. # distributed under the License is distributed on an "AS IS" BASIS,
  12. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. # See the License for the specific language governing permissions and
  14. # limitations under the License.
  15. from functools import wraps
  16. import random
  17. import sys
  18. from os import path
  19. import os
  20. import time
  21. from calculate.core.server.gen_pid import search_worked_process
  22. from calculate.core.setup_cache import Cache as SetupCache
  23. from calculate.core.server.func import MethodsInterface
  24. from calculate.lib.cl_template import SystemIni, LayeredIni
  25. from calculate.lib.datavars import DataVarsError, VariableError, Variable
  26. from calculate.lib.utils.tools import AddonError
  27. from calculate.lib.utils.colortext.palette import TextState
  28. from calculate.lib.utils.colortext import get_color_print
  29. from calculate.update.emerge_parser import RevdepPercentBlock
  30. from calculate.update.datavars import DataVarsUpdate
  31. from calculate.update.update_info import UpdateInfo
  32. from calculate.lib.utils.binhosts import (Binhosts, BinhostSignError,
  33. BinhostError, PackagesIndex, DAYS)
  34. from calculate.lib.utils.gpg import GPG, GPGError
  35. from calculate.lib.cl_log import log
  36. import hashlib
  37. import re
  38. import shutil
  39. from collections import MutableSet
  40. from contextlib import contextmanager
  41. import tempfile
  42. from fnmatch import fnmatch
  43. from calculate.lib.utils.vardb import Vardb
  44. from calculate.lib.utils.git import Git, GitError, MTimeKeeper, NotGitError
  45. from calculate.lib.utils.portage import (ReposConf, EmergeLog,
  46. EmergeLogNamedTask,
  47. VDB_PATH,
  48. PackageInformation,
  49. PortageState,
  50. get_packages_files_directory,
  51. get_manifest_files_directory,
  52. get_remove_list)
  53. from calculate.lib.utils.text import _u8, _u
  54. Colors = TextState.Colors
  55. from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
  56. getParentPids,
  57. PercentProgress, process, getRunCommands,
  58. readFile, listDirectory, pathJoin,
  59. find, FindFileType,quite_unlink,
  60. writeFile, makeDirectory)
  61. import emerge_parser
  62. import logging
  63. from emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
  64. EmergeCache, Chroot)
  65. from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
  66. RegexpLocalization, _)
  67. setLocalTranslate('cl_update3', sys.modules[__name__])
  68. __ = getLazyLocalTranslate(_)
  69. class UpdateError(AddonError):
  70. """Update Error"""
  71. class OverlayOwnCache(MutableSet):
  72. """
  73. Сет оверлеев с интегрированным кэшем
  74. """
  75. def __init__(self, dv=None):
  76. self.dv = dv
  77. def __get_overlays(self):
  78. own_cache_value = SystemIni(self.dv).getVar('system', 'own_cache') or ""
  79. return [x.strip() for x in own_cache_value.split(',') if x.strip()]
  80. def __write_overlays(self, overlays):
  81. if not overlays:
  82. SystemIni(self.dv).delVar('system', 'own_cache')
  83. else:
  84. SystemIni(self.dv).setVar('system',
  85. {'own_cache': ",".join(overlays)})
  86. def __contains__(self, item):
  87. return item in self.__get_overlays()
  88. def __iter__(self):
  89. return iter(self.__get_overlays())
  90. def __len__(self):
  91. return len(self.__get_overlays())
  92. def __append_value(self, overlays, value):
  93. if value not in overlays:
  94. overlays.append(value)
  95. self.__write_overlays(overlays)
  96. def add(self, value):
  97. overlays = self.__get_overlays()
  98. self.__append_value(overlays, value)
  99. def discard(self, value):
  100. overlays = self.__get_overlays()
  101. if value in overlays:
  102. overlays.remove(value)
  103. self.__write_overlays(overlays)
  104. def variable_module(var_env):
  105. def variable_module_decor(f):
  106. @wraps(f)
  107. def wrapper(self, *args, **kw):
  108. old_env = self.clVars.defaultModule
  109. try:
  110. self.clVars.defaultModule = var_env
  111. return f(self, *args, **kw)
  112. finally:
  113. self.clVars.defaultModule = old_env
  114. return wrapper
  115. return variable_module_decor
  116. class Update(MethodsInterface):
  117. """Основной объект для выполнения действий связанных с обновлением системы
  118. """
  119. def init(self):
  120. commandLog = path.join(self.clVars.Get('core.cl_log_path'),
  121. 'lastcommand.log')
  122. emerge_parser.CommandExecutor.logfile = commandLog
  123. self.color_print = get_color_print()
  124. self.emerge_cache = EmergeCache()
  125. if self.clVars.Get('cl_env_debug_set') == 'off':
  126. EmergeCache.logger.logger.setLevel(logging.WARNING)
  127. self.emerge_cache.check_list = (
  128. self.emerge_cache.check_list +
  129. map(lambda x:emerge_parser.GitCheckvalue(x, self.getGit()),
  130. self.clVars.Get('update.cl_update_rep_path')))
  131. self.update_map = {}
  132. self.refresh_binhost = False
  133. self.pkgnum = None
  134. self.pkgnummax = None
  135. self.gpgdata_md5 = []
  136. self.gpg_changed = False
  137. def get_prog_path(self, program_name):
  138. return getProgPath(program_name)
  139. def getGit(self):
  140. return self.clVars.Get('cl_update_git')
  141. @contextmanager
  142. def private_repo(self, rpath, url):
  143. if Git.is_private_url(url):
  144. try:
  145. if not path.exists(rpath):
  146. makeDirectory(rpath)
  147. os.chmod(rpath, 0700)
  148. yield
  149. finally:
  150. try:
  151. for dn in (Git._gitDir(rpath), path.join(rpath, "profiles/templates")):
  152. if path.exists(dn):
  153. os.chmod(dn, 0700)
  154. for fn in find(path.join(rpath, "profiles"), True, FindFileType.RegularFile,
  155. True, None, downfilter=lambda x: not x.endswith("/templates")):
  156. if fn.endswith("calculate.env") or fn.endswith("ini.env"):
  157. os.chmod(fn, 0600)
  158. if path.exists(rpath):
  159. os.chmod(rpath, 0755)
  160. except OSError:
  161. pass
  162. else:
  163. yield
  164. def _syncRepository(self, name, url, rpath, revision,
  165. cb_progress=None, clean=False, notask=False):
  166. """
  167. Синхронизировать репозитори
  168. """
  169. dv = self.clVars
  170. git = self.getGit()
  171. info_outdated = False
  172. old_dir = "%s.old" % git._gitDir(rpath)
  173. if path.exists(old_dir):
  174. clean = True
  175. try:
  176. self.stash_cache(rpath, name)
  177. if not git.checkExistsRep(rpath):
  178. if not notask:
  179. self.startTask(_("Syncing the {rep} repository").format(
  180. rep=name.capitalize()))
  181. self.addProgress()
  182. with self.private_repo(rpath, url):
  183. git.cloneTagRepository(url, rpath, revision,
  184. cb_progress=cb_progress)
  185. info_outdated = True
  186. else:
  187. cr = ""
  188. try:
  189. need_update = False
  190. tag_cr = git.getCommit(rpath, revision)
  191. cr = git.getCurrentCommit(rpath)
  192. ref_type = git.reference_type(rpath, revision)
  193. if tag_cr != cr or ref_type == Git.Reference.Branch:
  194. need_update = True
  195. elif clean:
  196. status = git.getStatusInfo(rpath)
  197. if not status or status['files']:
  198. need_update = True
  199. except GitError as e:
  200. need_update = True
  201. if need_update:
  202. if not notask:
  203. self.startTask(_("Syncing the {rep} repository").format(
  204. rep=name.capitalize()))
  205. self.addProgress()
  206. with self.private_repo(rpath, url):
  207. git.updateTagRepository(url, rpath, revision,
  208. cb_progress=cb_progress,
  209. clean=clean)
  210. new_cr = git.getCurrentCommit(rpath)
  211. if new_cr != cr:
  212. info_outdated = True
  213. if info_outdated:
  214. self.raiseOutdate()
  215. dv.Set('cl_update_outdate_set', 'on', force=True)
  216. finally:
  217. self.unstash_cache(rpath, name)
  218. # TODO: debug1
  219. #dv.Set('cl_update_outdate_set', 'on', force=True)
  220. return True
  221. def raiseOutdate(self):
  222. self.clVars.Set('cl_update_outdate_set', 'on', force=True)
  223. def setAutocheckParams(self, status, interval, update_other, cleanpkg):
  224. """
  225. Настроить параметры автопроверки обновлений
  226. """
  227. onoff = lambda x: "on" if x else "off"
  228. self.clVars.Write('cl_update_autocheck_set', onoff(status), True)
  229. self.clVars.Write('cl_update_autocheck_interval', interval, True)
  230. self.clVars.Write('cl_update_other_set', onoff(update_other), True)
  231. self.clVars.Write('cl_update_cleanpkg_set', onoff(cleanpkg), True)
  232. if not status:
  233. UpdateInfo.set_update_ready(False)
  234. return True
  235. def checkSchedule(self, interval, status):
  236. """
  237. Проверить по расписанию необходимость запуска команды
  238. """
  239. if not status:
  240. self.printWARNING(_("Updates autocheck is not enabled"))
  241. return False
  242. last_check = SystemIni(self.clVars).getVar('system', 'last_check') or ""
  243. re_interval = re.compile("^(\d+)\s*(hours?|days?|weeks?)?", re.I)
  244. interval_match = re_interval.search(interval)
  245. MINUTE = 60
  246. HOUR = MINUTE * 60
  247. DAY = HOUR * 24
  248. WEEK = DAY * 7
  249. if interval_match:
  250. if interval_match.group(2):
  251. suffix_map = {'h': HOUR, 'd': DAY, 'w': WEEK}
  252. k = suffix_map.get(interval_match.group(2).lower()[0], HOUR)
  253. else:
  254. k = HOUR
  255. est = int(interval_match.group(1)) * k
  256. else:
  257. est = 3 * HOUR
  258. if last_check:
  259. if last_check.isdigit():
  260. if (time.time() - int(last_check)) < (est - 10 * MINUTE):
  261. self.printWARNING(_("Please wait for the update time"))
  262. return False
  263. self.mark_schedule()
  264. return True
  265. def checkRun(self, wait_update):
  266. """
  267. Проверить повторный запуск
  268. """
  269. update_running = lambda: any(os.getpid() != x
  270. for x in
  271. search_worked_process('update', dv))
  272. dv = self.clVars
  273. if update_running():
  274. if not wait_update:
  275. raise UpdateError(_("Update is already running. "
  276. "Try to run later."))
  277. else:
  278. self.startTask(_("Waiting for another update to be complete"))
  279. while update_running():
  280. self.pauseProcess()
  281. while update_running():
  282. time.sleep(0.3)
  283. self.resumeProcess()
  284. time.sleep(random.random() * 3)
  285. self.endTask()
  286. if self.clVars.Get('cl_chroot_status') == 'off':
  287. parents_pids = set(getParentPids(os.getpid()))
  288. emerge_running = lambda : (
  289. any((fnmatch(x[1], "*python-exec/*/emerge*") and
  290. x[0] not in parents_pids)
  291. for x in getRunCommands(True, withpid=True)))
  292. if emerge_running():
  293. if not wait_update:
  294. raise UpdateError(_("Emerge is running. "
  295. "Try to run later."))
  296. else:
  297. self.startTask(_("Waiting for emerge to be complete"))
  298. while emerge_running():
  299. time.sleep(1)
  300. self.endTask()
  301. return True
  302. @variable_module("update")
  303. def trimRepositories(self, repname):
  304. """
  305. Синхронизировать репозитории
  306. """
  307. dv = self.clVars
  308. rpath = \
  309. dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1)
  310. git = self.getGit()
  311. self.addProgress()
  312. git.trimRepository(rpath, cb_progress=self.setProgress)
  313. return True
  314. def migrateLayman(self, reposdir, laymandir, repname, rpath_orig):
  315. if rpath_orig.startswith("/var/db/repos/"):
  316. dn_name = os.path.basename(rpath_orig)
  317. repos_fullname = pathJoin(reposdir, dn_name)
  318. layman_fullname = pathJoin(laymandir, dn_name)
  319. if (not os.path.islink(layman_fullname) and
  320. os.path.isdir(layman_fullname) and
  321. not os.path.exists(repos_fullname)):
  322. self.startTask(_("Move {repname} from {laymandir} to {reposdir}").format(
  323. repname=repname.capitalize(),
  324. laymandir="/var/lib/layman",
  325. reposdir="/var/db/repos"))
  326. symlink_target = os.path.relpath(repos_fullname, laymandir)
  327. if not os.path.exists(reposdir):
  328. makeDirectory(reposdir)
  329. os.rename(layman_fullname, repos_fullname)
  330. os.symlink(symlink_target, layman_fullname)
  331. #print "MYDEBUG", reposdir, laymandir, rpath_orig
  332. self.endTask(True)
  333. return True
  334. @variable_module("update")
  335. def syncRepositories(self, repname, fallback_sync=False,
  336. clean_on_error=True):
  337. """
  338. Синхронизировать репозитории
  339. """
  340. dv = self.clVars
  341. check_status = dv.GetBool('update.cl_update_check_rep_set')
  342. url, rpath, revision = (
  343. dv.Select(["cl_update_rep_url", "cl_update_rep_path",
  344. "cl_update_rep_rev"],
  345. where="cl_update_rep_name", eq=repname, limit=1))
  346. if not url or not rpath:
  347. raise UpdateError(_("Configuration variables for repositories "
  348. "are not setup"))
  349. git = self.getGit()
  350. if not git.checkUrl(url):
  351. raise UpdateError(_("Git %s is unavailable") % url)
  352. chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
  353. if chroot_path == '/':
  354. rpath_orig = rpath
  355. else:
  356. rpath_orig = rpath[len(chroot_path):]
  357. mtime = MTimeKeeper(path.join(rpath, "profiles/updates"))
  358. mtime.save()
  359. self.migrateLayman(dv.Get('cl_update_repos_storage'),
  360. dv.Get('cl_update_layman_storage'), repname, rpath_orig)
  361. try:
  362. if clean_on_error:
  363. try:
  364. repconf = ReposConf(dv.Get('cl_update_reposconf'),
  365. dv.Get('cl_update_reposconf_dir'),
  366. prefix=chroot_path)
  367. repconf.add(repname, url, rpath_orig)
  368. if not self._syncRepository(repname, url, rpath, revision,
  369. cb_progress=self.setProgress,
  370. clean=check_status,
  371. notask=fallback_sync):
  372. return "skip"
  373. return True
  374. except GitError as e:
  375. if not isinstance(e, NotGitError):
  376. if e.addon:
  377. self.printWARNING(str(e.addon))
  378. self.printWARNING(str(e))
  379. self.endTask(False)
  380. self.startTask(
  381. _("Re-fetching the {name} repository").format(
  382. name=repname))
  383. self.addProgress()
  384. rpath_new = "%s_new" % rpath
  385. try:
  386. self._syncRepository(repname, url, rpath_new, revision,
  387. cb_progress=self.setProgress,
  388. clean=check_status,
  389. notask=fallback_sync)
  390. removeDir(rpath)
  391. shutil.move(rpath_new, rpath)
  392. except OSError as e:
  393. raise UpdateError(_("Failed to modify the "
  394. "{repname} repository").format(
  395. repname=repname) + _(": ") + str(e))
  396. finally:
  397. if path.exists(rpath_new):
  398. removeDir(rpath_new)
  399. else:
  400. if not self._syncRepository(repname, url, rpath, revision,
  401. clean=check_status):
  402. return "skip"
  403. repconf = ReposConf(dv.Get('cl_update_reposconf'),
  404. dv.Get('cl_update_reposconf_dir'),
  405. prefix=chroot_path)
  406. repconf.add(repname, url, rpath_orig)
  407. finally:
  408. mtime.restore()
  409. return True
  410. metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
  411. def stash_cache(self, rpath, name):
  412. """
  413. Спрятать кэш
  414. """
  415. if name in ("portage", "gentoo"):
  416. return
  417. if not name in OverlayOwnCache(self.clVars):
  418. for cachename in self.metadata_cache_names:
  419. cachedir = path.join(rpath, cachename)
  420. if path.exists(cachedir):
  421. try:
  422. cachedir_s = path.join(path.dirname(rpath),
  423. path.basename(
  424. cachename) + ".stash")
  425. if path.exists(cachedir_s):
  426. removeDir(cachedir_s)
  427. shutil.move(cachedir, cachedir_s)
  428. except BaseException as e:
  429. pass
  430. def unstash_cache(self, rpath, name):
  431. """
  432. Извлеч кэш
  433. """
  434. if name in ("portage", "gentoo"):
  435. return
  436. cachenames = self.metadata_cache_names
  437. if not name in OverlayOwnCache(self.clVars):
  438. if any(path.exists(path.join(rpath, x)) for x in cachenames):
  439. for cachename in cachenames:
  440. cachedir_s = path.join(path.dirname(rpath),
  441. path.basename(cachename) + ".stash")
  442. if path.exists(cachedir_s):
  443. try:
  444. removeDir(cachedir_s)
  445. except BaseException as e:
  446. pass
  447. OverlayOwnCache(self.clVars).add(name)
  448. else:
  449. for cachename in cachenames:
  450. cachedir = path.join(rpath, cachename)
  451. cachedir_s = path.join(path.dirname(rpath),
  452. path.basename(cachename) + ".stash")
  453. if path.exists(cachedir_s):
  454. try:
  455. shutil.move(cachedir_s, cachedir)
  456. except BaseException as e:
  457. pass
  458. else:
  459. if all(not path.exists(path.join(rpath, x)) for x in cachenames):
  460. OverlayOwnCache(self.clVars).discard(name)
  461. def syncOtherRepository(self, repname):
  462. """
  463. Обновить репозиторий через emerge --sync
  464. """
  465. emerge = self.get_prog_path('/usr/bin/emerge')
  466. if not emerge:
  467. raise UpdateError(_("The Emerge is not found"))
  468. rpath = self.clVars.Select('cl_update_other_rep_path',
  469. where='cl_update_other_rep_name', eq=repname,
  470. limit=1)
  471. repdirname = path.basename(rpath)
  472. self.stash_cache(rpath, repdirname)
  473. try:
  474. if Git.is_git(rpath):
  475. self.addProgress()
  476. p = PercentProgress(emerge, "--sync", repname, part=1, atty=True)
  477. for perc in p.progress():
  478. self.setProgress(perc)
  479. else:
  480. p = process(emerge, "--sync", repname, stderr=STDOUT)
  481. if p.failed():
  482. raise UpdateError(
  483. _("Failed to update the {rname} repository").format(
  484. rname=repname),
  485. addon=p.read())
  486. finally:
  487. self.unstash_cache(rpath, repdirname)
  488. return True
  489. def _regenCache_process(self, progname, repname, cpu_num):
  490. return process(progname, "--repo=%s" % repname, "--update",
  491. "--jobs=%s" % cpu_num, stderr=STDOUT)
  492. def regenCache(self, repname):
  493. """
  494. Обновить кэш метаданных репозитория
  495. """
  496. egenCache = self.get_prog_path('/usr/bin/egencache')
  497. if not egenCache:
  498. raise UpdateError(_("The Portage tool is not found"))
  499. if repname in self.clVars.Get('cl_update_rep_name'):
  500. path_rep = self.clVars.Select('cl_update_rep_path',
  501. where='cl_update_rep_name',
  502. eq=repname, limit=1)
  503. repo_name = readFile(
  504. path.join(path_rep, "profiles/repo_name")).strip()
  505. if repo_name != repname:
  506. self.printWARNING(
  507. _("Repository '{repo_name}' called '{repname}'"
  508. " in cl_update_rep_name").format(
  509. repo_name=repo_name, repname=repname))
  510. raise UpdateError(_("Failed to update the cache of the {rname} "
  511. "repository").format(rname=repname))
  512. cpu_num = self.clVars.Get('hr_cpu_num')
  513. if repname in OverlayOwnCache(self.clVars):
  514. self.printWARNING(
  515. _("Repository %s has its own cache") % repname.capitalize())
  516. else:
  517. self.startTask(_("Updating the %s repository cache") %
  518. repname.capitalize())
  519. p = self._regenCache_process(egenCache, repname, cpu_num)
  520. if p.failed():
  521. raise UpdateError(_("Failed to update the cache of the {rname} "
  522. "repository").format(rname=repname),
  523. addon=p.read())
  524. return True
  525. def emergeMetadata(self):
  526. """
  527. Выполнить egencache и emerge --metadata
  528. """
  529. emerge = self.get_prog_path("/usr/bin/emerge")
  530. if not emerge:
  531. raise UpdateError(_("The Emerge tool is not found"))
  532. self.addProgress()
  533. p = PercentProgress(emerge, "--ask=n", "--metadata", part=1, atty=True)
  534. for perc in p.progress():
  535. self.setProgress(perc)
  536. if p.failed():
  537. data = p.read()
  538. with open('/var/log/calculate/failed-metadata-%d.log' % time.time(),
  539. 'w') as f:
  540. f.write(data + p.alldata)
  541. raise UpdateError(_("Failed to update metadata"), addon=data)
  542. return True
  543. def _eixUpdateCommand(self, eix_cmd, countRep):
  544. return PercentProgress(eix_cmd, "-F", part=countRep or 1, atty=True)
  545. def _copyPreviousEix(self):
  546. chroot_path = self.clVars.Get("cl_chroot_path")
  547. portage_eix = pathJoin(chroot_path, "var/cache/eix/portage.eix")
  548. previous_eix = pathJoin(chroot_path, "var/cache/eix/previous.eix")
  549. if path.exists(portage_eix):
  550. if path.exists(previous_eix):
  551. os.unlink(previous_eix)
  552. try:
  553. with writeFile(previous_eix) as f:
  554. f.write(readFile(portage_eix))
  555. portage_eix_stat = os.stat(portage_eix)
  556. os.chmod(previous_eix, portage_eix_stat.st_mode)
  557. os.chown(previous_eix, portage_eix_stat.st_uid,
  558. portage_eix_stat.st_gid)
  559. except (OSError,IOError):
  560. self.printWARNING(_("Failed to create previous eix"))
  561. return True
  562. def eixUpdate(self, repositories):
  563. """
  564. Выполенине eix-update для репозиторием
  565. eix-update выполнятется только для тех репозиториев, которые
  566. обновлялись, если cl_update_eixsync_force==auto, либо
  567. все, если cl_update_eixupdate_force==force
  568. """
  569. self._copyPreviousEix()
  570. eixupdate = self.get_prog_path("/usr/bin/eix-update")
  571. if not eixupdate:
  572. raise UpdateError(_("The Eix tool is not found"))
  573. self.addProgress()
  574. countRep = len(repositories)
  575. p = self._eixUpdateCommand(eixupdate, countRep)
  576. for perc in p.progress():
  577. self.setProgress(perc)
  578. if p.failed():
  579. raise UpdateError(_("Failed to update eix cache"), addon=p.read())
  580. return True
  581. def is_binary_pkg(self, pkg, binary=None):
  582. """
  583. Является ли пакет бинарным
  584. """
  585. if binary:
  586. return True
  587. if 'PN' in pkg and pkg['PN'].endswith('-bin'):
  588. return True
  589. if binary is not None:
  590. return binary
  591. if "binary" in pkg and pkg['binary']:
  592. return True
  593. return False
  594. def _printEmergePackage(self, pkg, binary=False, num=1, max_num=1):
  595. """
  596. Вывод сообщения сборки пакета
  597. """
  598. self.endTask()
  599. _print = self.color_print
  600. if self.pkgnum is not None:
  601. num = self.pkgnum
  602. if self.pkgnummax is not None:
  603. max_num = self.pkgnummax
  604. one = _print("{0}", num)
  605. two = _print("{0}", max_num)
  606. part = _("({current} of {maximum})").format(current=one, maximum=two)
  607. _print = _print.foreground(Colors.DEFAULT)
  608. if self.is_binary_pkg(pkg, binary):
  609. _colorprint = _print.foreground(Colors.PURPLE)
  610. else:
  611. _colorprint = _print.foreground(Colors.GREEN)
  612. PackageInformation.add_info(pkg)
  613. name = ""
  614. if pkg.info['DESCRIPTION']:
  615. name = _(pkg.info['DESCRIPTION'])
  616. name = name[:1].upper() + name[1:]
  617. if not name.strip():
  618. name = str(pkg)
  619. self.printSUCCESS(
  620. _u(_("{part} {package}")).format(part=_u(part), package=_u(name)))
  621. self.startTask(
  622. _u(_("Emerging {package}")).format(package=_u(_colorprint(str(pkg)))))
  623. def _printInstallPackage(self, pkg, binary=False):
  624. """
  625. Вывод сообщения установки пакета
  626. """
  627. self.endTask()
  628. _print = self.color_print
  629. if self.is_binary_pkg(pkg, binary):
  630. _print = _print.foreground(Colors.PURPLE)
  631. else:
  632. _print = _print.foreground(Colors.GREEN)
  633. pkg_key = "{CATEGORY}/{PF}".format(**pkg)
  634. if pkg_key in self.update_map:
  635. self.startTask(_("Installing {pkg} [{oldver}]").format(
  636. pkg=_print(str(pkg)), oldver=self.update_map[pkg_key]))
  637. self.update_map.pop(pkg_key)
  638. else:
  639. self.startTask(_("Installing %s") % (_print(str(pkg))))
  640. def _printFetching(self, fn):
  641. """
  642. Вывод сообщения о скачивании
  643. """
  644. self.endTask()
  645. self.startTask(_("Fetching binary packages"))
  646. def _printUninstallPackage(self, pkg, num=1, max_num=1):
  647. """
  648. Вывод сообщения удаления пакета
  649. """
  650. self.endTask()
  651. _print = self.color_print
  652. if num and max_num:
  653. one = _print("{0}", num)
  654. two = _print("{0}", max_num)
  655. part = _(" ({current} of {maximum})").format(current=one,
  656. maximum=two)
  657. else:
  658. part = ""
  659. _print = _print.foreground(Colors.LIGHT_RED)
  660. self.startTask(
  661. _("Unmerging{part} {package}").format(part=part,
  662. package=_print(str(pkg))))
  663. def emergelike(self, cmd, *params):
  664. """
  665. Запуск команды, которая подразумевает выполнение emerge
  666. """
  667. cmd_path = self.get_prog_path(cmd)
  668. if not cmd_path:
  669. raise UpdateError(_("Failed to find the %s command") % cmd)
  670. with EmergeParser(
  671. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  672. self._startEmerging(emerge)
  673. return True
  674. def revdep_rebuild(self, cmd, *params):
  675. """
  676. Запуск revdep-rebulid
  677. """
  678. cmd_path = self.get_prog_path(cmd)
  679. if not cmd_path:
  680. raise UpdateError(_("Failed to find the %s command") % cmd)
  681. with EmergeParser(
  682. emerge_parser.CommandExecutor(cmd_path, params)) as emerge:
  683. revdep = RevdepPercentBlock(emerge)
  684. self.addProgress()
  685. revdep.add_observer(self.setProgress)
  686. revdep.action = lambda x: (
  687. self.endTask(), self.startTask(_("Assigning files to packages"))
  688. if "Assign" in revdep else None)
  689. self._startEmerging(emerge)
  690. return True
  691. def _display_pretty_package_list(self, pkglist, remove_list=False):
  692. """
  693. Отобразить список пакетов в "удобочитаемом" виде
  694. """
  695. _print = self.color_print
  696. ebuild_color = TextState.Colors.GREEN
  697. binary_color = TextState.Colors.PURPLE
  698. remove_color = TextState.Colors.LIGHT_RED
  699. flag_map = {"updating":
  700. _print.foreground(TextState.Colors.LIGHT_CYAN)("U"),
  701. "reinstall":
  702. _print.foreground(TextState.Colors.YELLOW)("rR"),
  703. "new":
  704. _print.foreground(TextState.Colors.LIGHT_GREEN)("N"),
  705. "newslot":
  706. _print.foreground(TextState.Colors.LIGHT_GREEN)("NS"),
  707. "downgrading": (
  708. _print.foreground(TextState.Colors.LIGHT_CYAN)("U") +
  709. _print.foreground(TextState.Colors.LIGHT_BLUE)("D"))}
  710. for pkg in sorted([PackageInformation.add_info(x) for x in
  711. pkglist],
  712. key=lambda y: y['CATEGORY/PN']):
  713. install_flag = ""
  714. if remove_list:
  715. pkgcolor = _print.foreground(remove_color)
  716. else:
  717. for flag in flag_map:
  718. if pkg[flag]:
  719. install_flag = "(%s) " % flag_map[flag]
  720. break
  721. if self.is_binary_pkg(pkg):
  722. pkgcolor = _print.foreground(binary_color)
  723. else:
  724. pkgcolor = _print.foreground(ebuild_color)
  725. if pkg.info['DESCRIPTION']:
  726. fullname = "%s " % _(pkg.info['DESCRIPTION'])
  727. fullname = fullname[:1].upper() + fullname[1:]
  728. else:
  729. fullname = ""
  730. shortname = pkgcolor("%s-%s" % (pkg["CATEGORY/PN"], pkg["PVR"]))
  731. if "SIZE" in pkg and pkg['SIZE'] and pkg["SIZE"] != "0 kB":
  732. size = " (%s)" % pkg["SIZE"]
  733. else:
  734. size = ""
  735. mult = _print.bold("*")
  736. self.printDefault(
  737. _u8("&nbsp;{mult} {fullname}{flag}{shortname}{size}").format(
  738. mult=_u8(mult), fullname=_u8(fullname), shortname=_u8(shortname),
  739. size=_u8(size),
  740. flag=_u8(install_flag)))
  741. def _display_install_package(self, emerge, emergelike=False):
  742. """
  743. Отобразить список устанавливаемых пакетов
  744. """
  745. # подробный список пакетов
  746. _print = self.color_print
  747. if emergelike:
  748. self.printPre(str(emerge.install_packages))
  749. else:
  750. pkglist = emerge.install_packages.list
  751. self.printSUCCESS(_print(
  752. _("Listing packages for installation")))
  753. self._display_pretty_package_list(pkglist)
  754. if emerge.install_packages.remove_list:
  755. self.printSUCCESS(_print(
  756. _("Listing packages for removal")))
  757. self._display_pretty_package_list(
  758. emerge.install_packages.remove_list, remove_list=True)
  759. if len(emerge.install_packages.list) > 0:
  760. install_mess = (_("{count} packages will be installed").format(
  761. count=len(emerge.install_packages.list)) + ", ")
  762. else:
  763. install_mess = ""
  764. if str(emerge.download_size) != "0 kB":
  765. self.printSUCCESS(_("{install}{size} will be downloaded").format(
  766. install=install_mess,
  767. size=str(emerge.download_size)))
  768. def _display_remove_list(self, emerge):
  769. """
  770. Отобразить список удаляемых пакетов
  771. """
  772. # подробный список пакетов
  773. if self.clVars.Get('update.cl_update_emergelist_set') == 'on':
  774. self.printPre(self._emerge_translate(
  775. emerge.uninstall_packages.verbose_result))
  776. else:
  777. _print = self.color_print
  778. pkglist = emerge.uninstall_packages.list
  779. self.printSUCCESS(_print.bold(
  780. _("Listing packages for removal")))
  781. self._display_pretty_package_list(pkglist, remove_list=True)
  782. def mark_schedule(self):
  783. """
  784. Установить отметку о запуске запланированной проверки
  785. """
  786. SystemIni(self.clVars).setVar('system', {'last_check': str(int(time.time()))})
  787. def get_default_emerge_opts(self, depclean=False):
  788. bdeps_val = self.clVars.Get('cl_update_with_bdeps_set')
  789. if bdeps_val == "auto":
  790. bdeps = " --with-bdeps-auto=y"
  791. elif bdeps_val == "on":
  792. bdeps = " --with-bdeps=y"
  793. else:
  794. bdeps = " --with-bdeps=n"
  795. return self.clVars.Get('cl_emerge_default_opts') + bdeps
  796. def premerge(self, param, *packages):
  797. """
  798. Вывести информацию об обновлении
  799. """
  800. deo = self.get_default_emerge_opts()
  801. param = [param, "-pv"]
  802. with EmergeParser(EmergeCommand(list(packages), emerge_default_opts=deo,
  803. extra_params=param)) as emerge:
  804. try:
  805. emerge.run()
  806. if not emerge.install_packages.list:
  807. self.printSUCCESS(_("The system is up to date"))
  808. self.set_need_update(False)
  809. return True
  810. emergelike = self.clVars.Get('cl_update_emergelist_set') == 'on'
  811. self._display_install_package(emerge, emergelike)
  812. if str(emerge.skipped_packages):
  813. self._display_error(emerge.skipped_packages)
  814. except EmergeError:
  815. self.set_need_update(False)
  816. self._display_install_package(emerge, emergelike=True)
  817. self._display_error(emerge.prepare_error)
  818. raise
  819. if self.clVars.Get('cl_update_pretend_set') == 'on':
  820. # установить кэш: есть обновления
  821. self.set_need_update()
  822. return True
  823. self.set_need_update(False)
  824. answer = self.askConfirm(
  825. _("Would you like to merge these packages?"), "yes")
  826. if answer == "no":
  827. raise KeyboardInterrupt
  828. return "yes"
  829. return True
  830. def set_need_update(self, val=True):
  831. """
  832. Установить флаг: есть обновления
  833. """
  834. if self.clVars.Get('update.cl_update_autocheck_set') == 'off':
  835. val = False
  836. UpdateInfo.set_update_ready(val)
  837. return True
  838. def _emerge_translate(self, s):
  839. """
  840. Перевести текст из emerge
  841. """
  842. return RegexpLocalization('cl_emerge').translate(str(s))
  843. def setUpToDateCache(self):
  844. """
  845. Установить кэш - "нет пакетов для обновления"
  846. """
  847. #self.updateCache(PackageList([]))
  848. self.set_need_update(False)
  849. return True
  850. def _startEmerging(self, emerge):
  851. """
  852. Настроить и выполнить emerge
  853. """
  854. if emerge.install_packages and emerge.install_packages.list:
  855. for pkg in emerge.install_packages.list:
  856. rv = pkg.get('REPLACING_VERSIONS', '')
  857. if rv:
  858. self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \
  859. rv.partition(":")[0]
  860. emerge.command.send("yes\n")
  861. emerge.emerging.add_observer(self._printEmergePackage)
  862. emerge.installing.add_observer(self._printInstallPackage)
  863. emerge.uninstalling.add_observer(self._printUninstallPackage)
  864. emerge.fetching.add_observer(self._printFetching)
  865. def cancel_observing_fetch(fn):
  866. emerge.fetching.clear_observers()
  867. emerge.fetching.add_observer(cancel_observing_fetch)
  868. try:
  869. emerge.run()
  870. except EmergeError:
  871. if emerge.emerging_error:
  872. self._display_error(emerge.emerging_error.log)
  873. else:
  874. self._display_error(emerge.prepare_error)
  875. raise
  876. def _display_error(self, error):
  877. lines_num = int(self.clVars.Get('update.cl_update_lines_limit'))
  878. error = "<br/>".join(str(error).split('<br/>')[-lines_num:])
  879. self.printPre(self._emerge_translate(error))
  880. def emerge(self, use, param, *packages):
  881. """
  882. Выполнить сборку пакета
  883. """
  884. deo = self.get_default_emerge_opts()
  885. if not packages:
  886. packages = [param]
  887. extra_params = None
  888. else:
  889. extra_params = [param]
  890. command = EmergeCommand(list(packages), emerge_default_opts=deo,
  891. extra_params=extra_params,
  892. use=use)
  893. if self.clVars.Get('cl_chroot_path') != '/':
  894. command = Chroot(self.clVars.Get('cl_chroot_path'), command)
  895. with EmergeParser(command) as emerge:
  896. try:
  897. emerge.question.action = lambda x: False
  898. emerge.run()
  899. if not emerge.install_packages.list:
  900. return True
  901. except EmergeError:
  902. self._display_error(emerge.prepare_error)
  903. raise
  904. self._startEmerging(emerge)
  905. return True
  906. def emerge_ask(self, pretend, *params):
  907. """
  908. Вывести информацию об обновлении
  909. """
  910. deo = self.get_default_emerge_opts()
  911. param = [x for x in params if x.startswith("-")]
  912. packages = [x for x in params if not x.startswith("-")]
  913. command = EmergeCommand(list(packages), emerge_default_opts=deo,
  914. extra_params=param)
  915. with EmergeParser(command) as emerge:
  916. try:
  917. emerge.question.action = lambda x: False
  918. emerge.run()
  919. if emerge.install_packages.list:
  920. emergelike = self.clVars.Get(
  921. 'update.cl_update_emergelist_set') == 'on'
  922. self._display_install_package(emerge, emergelike)
  923. if emerge.skipped_packages:
  924. self._display_error(emerge.skipped_packages)
  925. if not pretend:
  926. answer = self.askConfirm(
  927. _("Would you like to merge these packages?"), "yes")
  928. if answer == "no":
  929. emerge.command.send("no\n")
  930. raise KeyboardInterrupt
  931. else:
  932. return True
  933. else:
  934. self.set_need_update(False)
  935. self.printSUCCESS(_("The system is up to date"))
  936. except EmergeError:
  937. self.set_need_update(False)
  938. self._display_install_package(emerge, emergelike=True)
  939. self._display_error(emerge.prepare_error)
  940. raise
  941. self._startEmerging(emerge)
  942. return True
  943. def depclean(self):
  944. """
  945. Выполнить очистку системы от лишних пакетов
  946. """
  947. deo = self.get_default_emerge_opts(depclean=True)
  948. emerge = None
  949. try:
  950. emerge = EmergeParser(EmergeCommand(["--depclean",
  951. "--dynamic-deps=n"],
  952. emerge_default_opts=deo))
  953. outdated_kernel = False
  954. try:
  955. emerge.question.action = lambda x: False
  956. emerge.run()
  957. if not emerge.uninstall_packages.list:
  958. UpdateInfo(self.clVars).outdated_kernel = False
  959. return True
  960. kernel_pkg = self.clVars.Get('cl_update_kernel_pkg')
  961. if any(("%s-%s" % (x['CATEGORY/PN'], x['PVR'])) == kernel_pkg
  962. for x in emerge.uninstall_packages.list):
  963. pkglist = [
  964. "=%s-%s" % (x['CATEGORY/PN'], x['PVR']) for x in
  965. emerge.uninstall_packages.list
  966. if ("%s-%s" % (x['CATEGORY/PN'],
  967. x['PVR'])) != kernel_pkg]
  968. emerge.command.send('n\n')
  969. emerge.close()
  970. emerge = None
  971. if not pkglist:
  972. UpdateInfo(self.clVars).outdated_kernel = True
  973. return True
  974. emerge = EmergeParser(
  975. EmergeCommand(pkglist,
  976. extra_params=["--unmerge", '--ask=y'],
  977. emerge_default_opts=deo))
  978. emerge.question.action = lambda x: False
  979. emerge.run()
  980. outdated_kernel = True
  981. else:
  982. outdated_kernel = False
  983. self._display_remove_list(emerge)
  984. except EmergeError:
  985. self._display_error(emerge.prepare_error)
  986. raise
  987. if (self.askConfirm(
  988. _("Would you like to unmerge these unused packages "
  989. "(recommended)?")) != 'yes'):
  990. return True
  991. UpdateInfo(self.clVars).outdated_kernel = outdated_kernel
  992. self._startEmerging(emerge)
  993. finally:
  994. if emerge:
  995. emerge.close()
  996. return True
  997. def update_task(self, task_name):
  998. """
  999. Декоратор для добавления меток запуска и останова задачи
  1000. """
  1001. def decor(f):
  1002. def wrapper(*args, **kwargs):
  1003. logger = EmergeLog(EmergeLogNamedTask(task_name))
  1004. logger.mark_begin_task()
  1005. ret = f(*args, **kwargs)
  1006. if ret:
  1007. logger.mark_end_task()
  1008. return ret
  1009. return wrapper
  1010. return decor
  1011. def migrateCacheRepository(self, url, branch, storage):
  1012. """
  1013. Перенести репозиторий из кэша в локальный
  1014. """
  1015. rep = storage.get_repository(url, branch)
  1016. if rep:
  1017. rep.storage = storage.storages[0]
  1018. self.clVars.Invalidate('cl_update_profile_storage')
  1019. return True
  1020. def reconfigureProfileVars(self, profile_dv, chroot):
  1021. """
  1022. Синхронизировать репозитории
  1023. Настройка переменных для выполнения синхронизации репозиториев
  1024. """
  1025. dv = self.clVars
  1026. try:
  1027. if not profile_dv:
  1028. raise UpdateError(
  1029. _("Failed to use the new profile. Try again."))
  1030. for var_name in ('cl_update_rep_path',
  1031. 'cl_update_rep_url',
  1032. 'cl_update_rep_name',
  1033. 'cl_update_branch',
  1034. 'cl_update_binhost_list',
  1035. 'cl_update_binhost_unstable_list',
  1036. 'cl_update_binhost_stable_set',
  1037. 'cl_update_binhost_stable_opt_set',
  1038. 'cl_update_branch_name',
  1039. 'cl_profile_system',
  1040. 'cl_update_rep'):
  1041. dv.Set(var_name, profile_dv.Get(var_name), force=True)
  1042. dv.Set('cl_chroot_path', chroot, force=True)
  1043. except DataVarsError as e:
  1044. error = UpdateError(_("Wrong profile"))
  1045. error.addon = e
  1046. raise error
  1047. return True
  1048. def setProfile(self, profile_shortname):
  1049. profile = self.clVars.Select('cl_update_profile_path',
  1050. where='cl_update_profile_shortname',
  1051. eq=profile_shortname, limit=1)
  1052. if not profile:
  1053. raise UpdateError(_("Failed to determine profile %s") %
  1054. self.clVars.Get('cl_update_profile_system'))
  1055. profile_path = path.relpath(profile, '/etc/portage')
  1056. try:
  1057. profile_file = '/etc/portage/make.profile'
  1058. if not path.exists(
  1059. path.join(path.dirname(profile_file), profile_path)):
  1060. raise UpdateError(
  1061. _("Failed to set the profile: %s") % _("Profile not found"))
  1062. for rm_fn in filter(path.lexists,
  1063. ('/etc/make.profile',
  1064. '/etc/portage/make.profile')):
  1065. os.unlink(rm_fn)
  1066. os.symlink(profile_path, profile_file)
  1067. except (OSError, IOError) as e:
  1068. raise UpdateError(_("Failed to set the profile: %s") % str(e))
  1069. return True
  1070. def applyProfileTemplates(self, useClt=None, cltFilter=False,
  1071. useDispatch=True, action="merge"):
  1072. """
  1073. Наложить шаблоны из профиля
  1074. """
  1075. from calculate.lib.cl_template import TemplatesError, ProgressTemplate
  1076. dv = DataVarsUpdate()
  1077. try:
  1078. dv.importUpdate()
  1079. dv.flIniFile()
  1080. dv.Set('cl_action', action, force=True)
  1081. try:
  1082. dv.Set('cl_templates_locate',
  1083. self.clVars.Get('cl_update_templates_locate'))
  1084. except VariableError:
  1085. self.printERROR(_("Failed to apply profiles templates"))
  1086. return True
  1087. dv.Set("cl_chroot_path", '/', True)
  1088. dv.Set("cl_root_path", '/', True)
  1089. for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
  1090. "update.cl_update_world"):
  1091. dv.Set(copyvar, self.clVars.Get(copyvar), True)
  1092. # определение каталогов содержащих шаблоны
  1093. useClt = useClt in ("on", True)
  1094. self.addProgress()
  1095. nullProgress = lambda *args, **kw: None
  1096. dispatch = self.dispatchConf if useDispatch else None
  1097. clTempl = ProgressTemplate(nullProgress, dv, cltObj=useClt,
  1098. cltFilter=cltFilter,
  1099. printSUCCESS=self.printSUCCESS,
  1100. printWARNING=self.printWARNING,
  1101. askConfirm=self.askConfirm,
  1102. dispatchConf=dispatch,
  1103. printERROR=self.printERROR)
  1104. try:
  1105. clTempl.applyTemplates()
  1106. if clTempl.hasError():
  1107. if clTempl.getError():
  1108. raise TemplatesError(clTempl.getError())
  1109. finally:
  1110. if clTempl:
  1111. if clTempl.cltObj:
  1112. clTempl.cltObj.closeFiles()
  1113. clTempl.closeFiles()
  1114. finally:
  1115. dv.close()
  1116. return True
  1117. def cleanpkg(self):
  1118. """
  1119. Очистить PKGDIR и DISTFILES в текущей системе
  1120. """
  1121. portdirs = ([self.clVars.Get('cl_portdir')] +
  1122. self.clVars.Get('cl_portdir_overlay'))
  1123. pkgfiles = get_packages_files_directory(*portdirs)
  1124. distdirfiles = get_manifest_files_directory(*portdirs)
  1125. distdir = self.clVars.Get('install.cl_distfiles_path')
  1126. pkgdir = self.clVars.Get('cl_pkgdir')
  1127. logger = log("update_cleanpkg.log",
  1128. filename="/var/log/calculate/update_cleanpkg.log",
  1129. formatter="%(asctime)s - %(clean)s - %(message)s")
  1130. return self._cleanpkg(
  1131. distdir, pkgdir, distdirfiles, pkgfiles, logger)
  1132. def _update_binhost_packages(self):
  1133. os.system('/usr/sbin/emaint binhost -f &>/dev/null')
  1134. def _cleanpkg(self, distdir, pkgdir, distdirfiles, pkgfiles, logger):
  1135. """
  1136. Общий алгоритм очистки distfiles и pkgdir от устаревших пакетов
  1137. """
  1138. skip_files = ["/metadata.dtd", "/Packages"]
  1139. try:
  1140. if self.clVars.Get('client.os_remote_auth'):
  1141. skip_files += ['portage_lockfile']
  1142. except DataVarsError:
  1143. pass
  1144. for cleantype, filelist in (
  1145. ("packages",
  1146. get_remove_list(pkgdir, list(pkgfiles), depth=4)),
  1147. ("distfiles",
  1148. get_remove_list(distdir, list(distdirfiles), depth=1))):
  1149. removelist = []
  1150. for fn in filelist:
  1151. try:
  1152. if not any(fn.endswith(x) for x in skip_files):
  1153. os.unlink(fn)
  1154. removelist.append(path.basename(fn))
  1155. except OSError:
  1156. pass
  1157. removelist_str = ",".join(removelist)
  1158. if removelist_str:
  1159. logger.info(removelist_str, extra={'clean': cleantype})
  1160. if cleantype == "packages":
  1161. try:
  1162. self._update_binhost_packages()
  1163. for dn in listDirectory(pkgdir, fullPath=True):
  1164. if path.isdir(dn) and not listDirectory(dn):
  1165. os.rmdir(dn)
  1166. except OSError:
  1167. pass
  1168. return True
  1169. def updateSetupCache(self):
  1170. cache = SetupCache(self.clVars)
  1171. cache.update(force=True)
  1172. return True
  1173. def get_bin_cache_filename(self):
  1174. return pathJoin(self.clVars.Get('cl_chroot_path'),
  1175. LayeredIni.IniPath.Grp)
  1176. def update_local_info_binhost(self, write_binhost=True):
  1177. """
  1178. Проверить, что доступен хотя бы один из binhost'ов
  1179. :return:
  1180. """
  1181. hosts = self.clVars.Get("update.cl_update_binhost_host")
  1182. datas = self.clVars.Get("update.cl_update_binhost_revisions")
  1183. if not hosts:
  1184. self.delete_binhost()
  1185. raise UpdateError(_("Failed to find the server with "
  1186. "appropriate updates"))
  1187. else:
  1188. with writeFile(self.get_bin_cache_filename()) as f:
  1189. f.write(datas[0].strip()+"\n")
  1190. if write_binhost:
  1191. if hosts[0] != self.clVars.Get('update.cl_update_binhost'):
  1192. self.refresh_binhost = True
  1193. self.clVars.Set('cl_update_package_cache_set', 'on')
  1194. self.clVars.Write('cl_update_binhost', hosts[0], location="system")
  1195. new_ts = self.clVars.Get("update.cl_update_binhost_timestamp")
  1196. if new_ts:
  1197. new_ts = new_ts[0]
  1198. if new_ts.isdigit():
  1199. ini = SystemIni(self.clVars)
  1200. ini.setVar('system', {'last_update': new_ts})
  1201. if self.is_update_action(self.clVars.Get("cl_action")):
  1202. value = self.clVars.GetBool('update.cl_update_binhost_stable_set')
  1203. new_value = self.clVars.GetBool('update.cl_update_binhost_stable_opt_set')
  1204. if value != new_value:
  1205. self.clVars.Write(
  1206. 'cl_update_binhost_stable_set',
  1207. self.clVars.Get('update.cl_update_binhost_stable_opt_set'),
  1208. location="system")
  1209. return True
  1210. def is_update_action(self, action):
  1211. return action == 'sync'
  1212. def modify_binary_depends(self, bdeps, prefix="/"):
  1213. """
  1214. Включить/выключить сборочные зависимости для бинарных пакетов
  1215. """
  1216. hide = bdeps == "auto"
  1217. self.startTask(_("Updating binary build dependences"))
  1218. vardb = Vardb(pathJoin(prefix,VDB_PATH))
  1219. if hide:
  1220. vardb.hide_binary_depends()
  1221. else:
  1222. vardb.unhide_binary_depends()
  1223. self.endTask()
  1224. return True
  1225. def save_with_bdeps(self):
  1226. oldval = self.clVars.Get('cl_update_with_bdeps_set')
  1227. newval = self.clVars.Get('cl_update_with_bdeps_opt_set')
  1228. if oldval != newval:
  1229. self.modify_binary_depends(newval)
  1230. self.clVars.Write('cl_update_with_bdeps_set', newval,
  1231. location="system")
  1232. self.clVars.Set('cl_update_force_depclean_set', 'on')
  1233. return True
  1234. def message_binhost_changed(self):
  1235. if self.refresh_binhost:
  1236. self.printWARNING(_("Update server was changed to %s") %
  1237. self.clVars.Get('update.cl_update_binhost'))
  1238. self.clVars.Set("update.cl_update_package_cache_set",
  1239. Variable.On, force=True)
  1240. else:
  1241. self.printSUCCESS(_("Update server %s") %
  1242. self.clVars.Get('update.cl_update_binhost'))
  1243. return True
  1244. def delete_binhost(self):
  1245. self.clVars.Delete('cl_update_binhost', location="system")
  1246. try:
  1247. bin_cache_fn = self.get_bin_cache_filename()
  1248. if path.exists(bin_cache_fn):
  1249. os.unlink(bin_cache_fn)
  1250. except OSError:
  1251. raise UpdateError(
  1252. _("Failed to remove cached ini.env of binary repository"))
  1253. try:
  1254. for varname in ('update.cl_update_package_cache',
  1255. 'update.cl_update_package_cache_sign'):
  1256. fn = self.clVars.Get(varname)
  1257. if path.exists(fn):
  1258. os.unlink(fn)
  1259. except OSError:
  1260. raise UpdateError(
  1261. _("Failed to remove cached Package index"))
  1262. # удалить binhost
  1263. binhost_fn = self.inchroot(
  1264. self.clVars.Get("update.cl_update_portage_binhost_path"))
  1265. if path.exists(binhost_fn):
  1266. os.unlink(binhost_fn)
  1267. return False
  1268. def update_binhost_list(self, dv=None):
  1269. """
  1270. Обновить список binhost'ов после обновления до master веток
  1271. :return:
  1272. """
  1273. changes = False
  1274. try:
  1275. if dv is None:
  1276. dv = DataVarsUpdate()
  1277. dv.importUpdate()
  1278. dv.flIniFile()
  1279. for varname in ('update.cl_update_binhost_list',
  1280. 'update.cl_update_binhost_unstable_list',
  1281. 'update.cl_update_binhost_timestamp_path',
  1282. 'update.cl_update_binhost_revision_path'):
  1283. new_value = dv.Get(varname)
  1284. old_value = self.clVars.Get(varname)
  1285. if new_value != old_value:
  1286. changes = True
  1287. self.clVars.Set(varname, new_value, force=True)
  1288. except DataVarsError as e:
  1289. raise UpdateError(_("Failed to get values for binhost search"))
  1290. if not changes:
  1291. return False
  1292. self.create_binhost_data()
  1293. return True
  1294. def drop_binhosts(self, dv):
  1295. """
  1296. Обновление до master веток
  1297. """
  1298. branch = dv.Get('update.cl_update_branch')
  1299. revs = [
  1300. branch for x in dv.Get('update.cl_update_rep_name')
  1301. ]
  1302. dv.Set('update.cl_update_branch_name', revs)
  1303. dv.Invalidate('update.cl_update_rep_rev')
  1304. return True
  1305. def inchroot(self, fn):
  1306. return pathJoin(self.clVars.Get("cl_chroot_path"), fn)
  1307. def prepare_gpg(self):
  1308. """
  1309. Получить объект для проверки подписи, либо получить заглушку
  1310. """
  1311. gpg_force = self.clVars.Get('cl_update_gpg_force')
  1312. gpg_keys = [self.inchroot(x)
  1313. for x in self.clVars.Get('cl_update_gpg_keys')]
  1314. if gpg_force == "skip":
  1315. return True
  1316. gpgtmpdir = "/var/calculate/tmp/update"
  1317. if not path.exists(gpgtmpdir):
  1318. makeDirectory(gpgtmpdir)
  1319. gpg = GPG(tempfile.mkdtemp(dir=gpgtmpdir,
  1320. prefix="gpg-"))
  1321. for keyfn in gpg_keys:
  1322. if path.exists(keyfn):
  1323. try:
  1324. key = readFile(keyfn)
  1325. gpg.import_key(key)
  1326. except GPGError as e:
  1327. self.printWARNING(_("Failed to load public keys from '%s' "
  1328. "for signature checking") % keyfn)
  1329. if not gpg.count_public():
  1330. if gpg_force == "force":
  1331. raise UpdateError(_("Public keys for Packages signature checking not found"))
  1332. else:
  1333. return True
  1334. self.clVars.Set('update.cl_update_gpg', gpg, force=True)
  1335. return True
  1336. def download_packages(self, url_binhost, packages_fn, packages_sign_fn, gpg):
  1337. quite_unlink(packages_fn)
  1338. orig_packages = Binhosts.fetch_packages(url_binhost)
  1339. try:
  1340. with writeFile(packages_fn) as f:
  1341. pi = PackagesIndex(orig_packages)
  1342. pi["TTL"] = str(30 * DAYS)
  1343. pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time()))
  1344. pi.write(f)
  1345. except (OSError, IOError):
  1346. raise UpdateError(_("Failed to save Packages"))
  1347. self.endTask(True)
  1348. self.startTask(_("Check packages index signature"))
  1349. if not gpg:
  1350. self.endTask("skip")
  1351. self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True)
  1352. return True
  1353. try:
  1354. Binhosts.check_packages_signature(
  1355. url_binhost, orig_packages, gpg)
  1356. with writeFile(packages_sign_fn) as f:
  1357. f.write(Binhosts.fetch_packages_sign(url_binhost))
  1358. except BinhostSignError:
  1359. for fn in (packages_fn, packages_sign_fn):
  1360. if path.exists(fn):
  1361. try:
  1362. os.unlink(fn)
  1363. except OSError:
  1364. pass
  1365. self.clVars.Set("update.cl_update_bad_sign_set", Variable.On)
  1366. self.clVars.Set('update.cl_update_binhost_recheck_set', Variable.On)
  1367. self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True)
  1368. raise
  1369. return True
  1370. class Reason(object):
  1371. Success = 0
  1372. BadSign = 1
  1373. Outdated = 2
  1374. Skip = 3
  1375. Updating = 4
  1376. WrongBinhost = 5
  1377. BadEnv = 6
  1378. EnvNotFound = 7
  1379. SkipSlower = 8
  1380. UnknownError = 9
  1381. @staticmethod
  1382. def humanReadable(reason):
  1383. return {
  1384. Update.Reason.WrongBinhost: "FAILED (Wrong binhost)",
  1385. Update.Reason.Outdated: "OUTDATED",
  1386. Update.Reason.Updating: "UPDATING",
  1387. Update.Reason.BadEnv: "FAILED (Bad env)",
  1388. Update.Reason.EnvNotFound: "FAILED (Env not found)",
  1389. Update.Reason.UnknownError: "FAILED (Unknown error)",
  1390. Update.Reason.BadSign: "FAILED (Bad sign)",
  1391. Update.Reason.Skip: "SKIP",
  1392. Update.Reason.SkipSlower: "",
  1393. Update.Reason.Success: ""
  1394. }.get(reason,reason)
  1395. def _get_binhost_logger(self):
  1396. return log("binhost-scan.log",
  1397. filename=pathJoin(
  1398. self.clVars.Get('cl_chroot_path'),
  1399. "/var/log/calculate/binhost-scan.log"),
  1400. formatter="%(message)s")
  1401. def get_arch_machine(self):
  1402. return self.clVars.Get('os_arch_machine')
  1403. @variable_module("update")
  1404. def create_binhost_data(self):
  1405. dv = self.clVars
  1406. last_ts = dv.Get('cl_update_last_timestamp')
  1407. if dv.GetBool('cl_update_binhost_stable_opt_set'):
  1408. binhost_list = dv.Get('cl_update_binhost_list')
  1409. else:
  1410. binhost_list = dv.Get('cl_update_binhost_unstable_list')
  1411. self.binhosts_data = Binhosts(
  1412. # значение малозначимо, поэтому берётся из собирающей системы
  1413. dv.GetInteger('cl_update_binhost_timeout'),
  1414. dv.Get('cl_update_binhost_revision_path'),
  1415. dv.Get('cl_update_binhost_timestamp_path'),
  1416. last_ts, binhost_list,
  1417. self.get_arch_machine(),
  1418. gpg=dv.Get('update.cl_update_gpg'))
  1419. return True
  1420. @variable_module("update")
  1421. def _search_best_binhost(self, binhosts_data, stabilization):
  1422. if not self.clVars.Get('cl_ebuild_phase'):
  1423. logger = self._get_binhost_logger()
  1424. if logger:
  1425. logger.info(
  1426. "Started scan on: {date}, current timestamp: {ts}".format(
  1427. date=time.ctime(), ts=binhosts_data.last_ts))
  1428. retval = []
  1429. skip_check_status = False
  1430. actual_reason = self.Reason.UnknownError
  1431. for binhost in sorted(binhosts_data.get_binhosts(), reverse=True):
  1432. host = binhost.host
  1433. if not binhost.valid:
  1434. reason = self.Reason.WrongBinhost
  1435. elif binhost.outdated:
  1436. reason = self.Reason.Outdated
  1437. elif not skip_check_status:
  1438. status = binhost.status
  1439. if status is not binhosts_data.BinhostStatus.Success:
  1440. errors = {
  1441. binhosts_data.BinhostStatus.Updating: self.Reason.Updating,
  1442. binhosts_data.BinhostStatus.BadEnv: self.Reason.BadEnv,
  1443. binhosts_data.BinhostStatus.EnvNotFound: self.Reason.EnvNotFound
  1444. }
  1445. reason = errors.get(status, self.Reason.UnknownError)
  1446. elif binhost.bad_sign:
  1447. reason = self.Reason.BadSign
  1448. else:
  1449. # SUCCESS
  1450. if not binhost.downgraded or stabilization:
  1451. host = "-> %s" % host
  1452. reason = self.Reason.Success
  1453. else:
  1454. reason = self.Reason.Skip
  1455. elif binhost.downgraded:
  1456. reason = self.Reason.Skip
  1457. else:
  1458. reason = self.Reason.SkipSlower
  1459. if reason == self.Reason.Success:
  1460. retval.append([binhost.host, binhost.data,
  1461. str(binhost.timestamp),
  1462. str(binhost.duration)])
  1463. skip_check_status = True
  1464. if reason < actual_reason:
  1465. actual_reason = reason
  1466. logger.info("{host:<60} {speed:<7} {timestamp:<10} {reason}".format(
  1467. host=host, speed=float(binhost.duration) / 1000.0,
  1468. timestamp=binhost.timestamp,
  1469. reason=Update.Reason.humanReadable(reason)))
  1470. if not retval:
  1471. if actual_reason is self.Reason.BadSign:
  1472. raise UpdateError(_("Failed to find the reliable server with appropriate updates"))
  1473. elif actual_reason in (self.Reason.Outdated,
  1474. self.Reason.Skip,
  1475. self.Reason.Updating):
  1476. raise UpdateError(_("Failed to find the server with appropriate updates"))
  1477. else:
  1478. raise UpdateError(_("Failed to find the working server with updates"))
  1479. return retval
  1480. def check_current_binhost(self, binhost_url):
  1481. """
  1482. Проверка текущего сервера обновлений на валидность
  1483. """
  1484. if not binhost_url in self.binhosts_data.binhost_list:
  1485. raise UpdateError(_("Current binhost is absent in list of update servers"))
  1486. binhost = self.binhosts_data.get_binhost(binhost_url)
  1487. if binhost.valid and not binhost.outdated and not binhost.downgraded:
  1488. if binhost.status == self.binhosts_data.BinhostStatus.Success:
  1489. self.clVars.Set('update.cl_update_binhost_data',
  1490. [[binhost.host, binhost.data,
  1491. str(binhost.timestamp),
  1492. str(binhost.duration)]],
  1493. force=True)
  1494. self.endTask()
  1495. else:
  1496. if not binhost.valid:
  1497. raise UpdateError(
  1498. _("Current binhost {} is not valid").format(binhost_url))
  1499. elif binhost.outdated:
  1500. raise UpdateError(
  1501. _("Current binhost {} is outdated").format(binhost_url))
  1502. elif binhost.downgraded:
  1503. raise UpdateError(
  1504. _("Binary packages on the current binhost {} "
  1505. "are older than local").format(binhost_url))
  1506. if self.binhosts_data.gpg:
  1507. packages_fn = self.clVars.Get('update.cl_update_package_cache')
  1508. packages_sign_fn = self.clVars.Get('update.cl_update_package_cache_sign')
  1509. if path.exists(packages_fn) and path.exists(packages_sign_fn):
  1510. packages_sign = readFile(packages_sign_fn)
  1511. pi = PackagesIndex(readFile(packages_fn))
  1512. pi.clean()
  1513. try:
  1514. Binhosts.check_packages_signature(
  1515. None, pi.get_value(), self.binhosts_data.gpg,
  1516. sign=packages_sign)
  1517. return True
  1518. except BinhostSignError:
  1519. for fn in (packages_fn, packages_sign_fn):
  1520. if path.exists(fn):
  1521. try:
  1522. os.unlink(fn)
  1523. except OSError:
  1524. pass
  1525. if binhost.bad_sign:
  1526. raise UpdateError(
  1527. _("Current binhost {} has wrong signature").format(
  1528. binhost_url))
  1529. return True
  1530. @variable_module("update")
  1531. def detect_best_binhost(self):
  1532. # выполняется переход с серверов unstable обновлней на stable
  1533. # в этом случае не важно, что бинари могут старее текущих
  1534. if (self.clVars.GetBool('cl_update_binhost_stable_opt_set') and
  1535. not self.clVars.GetBool('cl_update_binhost_stable_set')):
  1536. stabilization = True
  1537. else:
  1538. stabilization = False
  1539. self.startTask(_("Searching new binhost"))
  1540. retval = self._search_best_binhost(self.binhosts_data, stabilization)
  1541. self.clVars.Set('update.cl_update_binhost_data',
  1542. retval or Variable.EmptyTable, force=True)
  1543. self.endTask()
  1544. return True
  1545. def update_rep_list(self):
  1546. """
  1547. Обновить список доступных репозиториев
  1548. :param builder_path:
  1549. :return:
  1550. """
  1551. cmd = "/usr/bin/eselect"
  1552. cmd_path = self.get_prog_path(cmd)
  1553. if not cmd_path:
  1554. raise UpdateError(_("Failed to find the %s command") % cmd)
  1555. repsync = emerge_parser.CommandExecutor(cmd_path, ["repository", "list"])
  1556. repsync.execute()
  1557. return repsync.success()
  1558. def rename_custom_files(self):
  1559. """
  1560. Переименовать все custom файлы: keywords, use, sets и т.д. в связи
  1561. с изменением профиля
  1562. """
  1563. newdv = self.clVars.Get("cl_update_profile_datavars")
  1564. cur_short = self.clVars.Get("os_linux_shortname").lower()
  1565. new_short = newdv["os_linux_shortname"].lower()
  1566. if cur_short != new_short:
  1567. for fn in ("/etc/portage/package.keywords/custom.{}",
  1568. "/etc/portage/package.use/custom.{}",
  1569. "/etc/portage/package.mask/custom.{}",
  1570. "/etc/portage/package.unmask/custom.{}",
  1571. "/etc/portage/sets/custom.{}",
  1572. "/etc/portage/make.conf/custom.{}"):
  1573. fn_source = fn.format(cur_short)
  1574. fn_target = fn.format(new_short)
  1575. try:
  1576. if path.exists(fn_source) and not path.exists(fn_target):
  1577. os.rename(fn_source, fn_target)
  1578. except (OSError, IOError) as e:
  1579. self.printWARNING(str(e))
  1580. world_sets_fn = "/var/lib/portage/world_sets"
  1581. if path.exists(world_sets_fn):
  1582. worlds_sets = readFile(world_sets_fn)
  1583. new_worlds_sets = re.sub("^@custom.{}$".format(cur_short),
  1584. "@custom.{}".format(new_short),
  1585. worlds_sets, flags=re.M)
  1586. try:
  1587. with open(world_sets_fn, 'w') as fd:
  1588. fd.write(new_worlds_sets)
  1589. except IOError as e:
  1590. self.printWARNING(str(e))
  1591. return True
  1592. def save_portage_state_hash(self):
  1593. """
  1594. Сохранить состояние
  1595. """
  1596. ini = SystemIni(self.clVars)
  1597. ps = PortageState()
  1598. new_portage_state_hash = ps.get_state()
  1599. ini.setVar('system', {'portage_hash': new_portage_state_hash})
  1600. return True