#-*- coding: utf-8 -*- # Copyright 2015 Calculate Ltd. http://www.calculate-linux.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from itertools import chain import random import sys import time import stat from calculate.core.server.gen_pid import search_worked_process2 from calculate.lib.datavars import DataVars from calculate.lib.utils.colortext import TextState, get_color_print, \ convert_console_to_xml from calculate.lib.utils.files import ( pathJoin, PercentProgress, getProgPath, process, STDOUT, removeDir, makeDirectory, writeFile, readLinesFile, chmod, chown, FilePermission, find, FindFileType, removeFileWithEmptyDirectory, copyWithPath, countFiles, listDirectory) from calculate.lib.utils.git import Git from calculate.lib.utils.portage import Layman, EmergeLog, EmergeLogNamedTask, \ InstalledPackageInfo, EbuildInfoError, EbuildInfo, ChrootEix, \ get_packages_files_directory, get_manifest_files_directory from calculate.update.emerge_parser import EmergeParser, \ EmergeError, EmergeCommand, Chroot, Linux32, CommandExecutor, \ RevdepPercentBlock, EmergeCache from calculate.lib.cl_log import log from .build_storage import Build from calculate.update.update import Update from calculate.install.distr import Distributive, IsoDistributive, \ DistributiveError from functools import partial import os from os import path from .datavars import BuilderError from .emerge_fetch import EmergeFetcher, EmergeFetcherError from calculate.install.distr import SquashDistributive from calculate.builder.variables.action import Actions from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, _) setLocalTranslate('cl_builder3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) class Builder(Update): """Основной объект для выполнения действий связанных со сборкой системы """ class Method: Prepare = "builder_prepare" Break = "builder_break" Update = "builder_update" Restore = "builder_restore" Image = "builder_image" Profile = "builder_profile" UpdateMenu = "builder_menu" All = (Prepare, Break, Update, Restore, Image, Profile) def init(self): self.pretend_package_list = {} self.update_map = {} self.color_print = get_color_print() self.emerge_cache = EmergeCache() def mount_target(self, target): dir_distro = target.convertToDirectory() dir_distro.mountSystemDirectories(skip=()) return True def umount_system(self, target): dir_distro = target.convertToDirectory() dir_distro.umountSystemDirectories() return True def detach_target(self, target): """ @type target: Distributive """ if target: target.reserve() return True def close_build(self, build, clear=False): """ @type build:Build """ if build: build.remove() build.close_distributive() if clear: build.distributive.post_clear() pkgdir = self.clVars.Get('cl_builder_pkgdir') if '/remote/' in pkgdir: if path.exists(pkgdir): removeDir(pkgdir) Distributive.clear_empty_directories(pkgdir) return True def restore_build(self, build): if build: build.restore() return True def save_build(self, build, dv): build.pkgdir = self.clVars.Get('cl_builder_pkgdir') if dv: overlays = dv.Get('cl_update_rep_name') build.set_overlays(overlays) build.save() return True def set_builder_action(self, action_name): self.clVars.Set('cl_builder_action', action_name, force=True) return True def prepare_iso(self, dn): self.endTask() self.startTask(_("Prepare ISO data")) root_path = path.relpath(dn, self.clVars.Get('cl_builder_path')) self.applyTemplates(self.clVars.Get('cl_builder_target'), False, False, root_path) self.endTask() self.startTask(_("Pack ISO image")) self.addProgress() def prepare_image(self, image): image.eventPrepareIso.connect(self.prepare_iso) return True def remove_repositories(self, repname): dv = self.clVars.Get('cl_builder_linux_datavars') rpath = dv.Select("cl_update_rep_path", where="cl_update_rep_name", eq=repname, limit=1) chroot_path = path.normpath(self.clVars.Get('cl_chroot_path')) rpath_orig = rpath[len(chroot_path):] layman = Layman(self.clVars.Get('update.cl_update_layman_installed'), self.clVars.Get('update.cl_update_layman_make'), self.clVars.Get('update.cl_update_layman_conf')) if repname != "portage": layman.remove(repname, rpath_orig) removeDir(rpath) return True def set_profile(self, profile_shortname, chroot_path): profile = self.clVars.Select('cl_builder_profile_path', where='cl_builder_profile_shortname', eq=profile_shortname, limit=1) if not profile: raise BuilderError(_("Failed to determine profile %s") % profile_shortname) chroot_join = lambda x: path.join(chroot_path, x) profile_path = path.relpath(profile, chroot_join('etc/portage')) try: for rm_fn in filter(path.lexists, (chroot_join('etc/make.profile'), chroot_join('etc/portage/make.profile'))): os.unlink(rm_fn) os.symlink(profile_path, chroot_join('etc/portage/make.profile')) except (OSError,IOError) as e: print str(e) raise BuilderError(_("Failed to set the profile: %s")%str(e)) return True def apply_templates(self,target=None,useClt=None,cltFilter=False,root=None, useDispatch=True, action="merge", distro_dv=None): """ Применить шаблоны. Args: target: дистрибутив, куда необходимо выполнить шаблоны (/ по умолчанию) useClt: использовать clt шаблоны cltFilter: применять фильтр на clt шаблоны root: каталог, куда будут наложны шаблоны (cl_root_path) """ from calculate.lib.cl_template import (TemplatesError, ProgressTemplate, templateFunction) templateFunction.installProg = {} templateFunction.installCategory = [] if target is None: chroot = '/' elif isinstance(target, Distributive): chroot = target.getDirectory() else: chroot = target if root is None: root = '/' elif isinstance(root, Distributive): root = root.getDirectory() clVars = DataVars() clTempl = None try: clVars.importData() if distro_dv: clVars.Set('cl_template_path', [pathJoin(chroot, x) for x in distro_dv.Get('cl_template_path')], force=True) clVars.Set('cl_template_path_use', clVars.Get('cl_template_path'), force=True) clVars.Set('cl_env_path', [pathJoin(chroot, x) for x in clVars.Get('cl_env_path')], force=True) clVars.Set('cl_make_profile', path.join(chroot, 'etc/portage/make.profile'), force=True) clVars.Set('cl_action', action, force=True) clVars.Set('cl_chroot_status', 'on', force=True) for copyvar in ("cl_dispatch_conf", "cl_verbose_set", "update.cl_update_world"): clVars.Set(copyvar, self.clVars.Get(copyvar), force=True) clVars.iniCache = {} clVars.flIniFile() cltFilter=True if cltFilter in (True,"on") else False clVars.Set("cl_chroot_path", chroot, True) clVars.Set("cl_root_path", root, True) # определение каталогов содержащих шаблоны dirs_list, files_list = ([],[]) useClt = useClt in ("on", True) self.addProgress() nullProgress = lambda *args,**kw:None dispatch = self.dispatchConf if useDispatch else None clTempl = ProgressTemplate(nullProgress,clVars, cltObj=useClt, cltFilter=cltFilter, printSUCCESS=self.printSUCCESS, printWARNING=self.printWARNING, askConfirm=self.askConfirm, dispatchConf=dispatch, printERROR=self.printERROR) def execute_command(cmd, lang): chroot_path = self.clVars.Get('cl_builder_path') env = dict(os.environ) env['TERM'] = "linux" env['EINFO_QUIET'] = "yes" return self.chroot_process( chroot_path, cmd, lang=lang, envdict=dict(os.environ)) # замена выполения команд: вместо оычного запуска - запуск через # /usr/bin/chroot clTempl.execute_command = execute_command clTempl.applyTemplates() if clTempl.hasError(): if clTempl.getError(): raise TemplatesError(clTempl.getError()) finally: clVars.close() if clTempl: if clTempl.cltObj: clTempl.cltObj.closeFiles() clTempl.closeFiles() return True def get_prog_path(self, progname): chroot_path = self.clVars.Get('builder.cl_builder_path') return getProgPath(progname, chroot_path) def _eixUpdateCommand(self, eix_cmd, countRep): chroot_path = self.clVars.Get('cl_builder_path') return PercentProgress("/usr/bin/chroot", chroot_path, eix_cmd, "-F", part=countRep or 1, atty=True) def regenCache(self, repname): with self.clVars.useDefaultModule("update"): return super(Builder, self).regenCache(repname) def syncRepositories(self, repname, clean_on_error=True): with self.clVars.useDefaultModule("update"): return super(Builder, self).syncRepositories(repname) def _regenCache_process(self, progname, repname, cpu_num): chroot_path = self.clVars.Get('builder.cl_builder_path') return self.chroot_process(chroot_path, progname, "--repo=%s" % repname, "--update", "--jobs=%s" % cpu_num, stderr=STDOUT) def clear_log(self, builder_id_path): logname = "build-%s" % builder_id_path mainlog = self.clVars.Get('core.cl_log_path') logpath = path.join(mainlog, logname) if path.exists(logpath): removeDir(logpath) makeDirectory(logpath) return True def _get_log_file(self): logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'), self.clVars.Get('cl_task_name')) mainlog = self.clVars.Get('core.cl_log_path') return path.join(mainlog, logname) def chrootize(self, chroot_path, cmd): """ :param chroot_path: :param cmd: :return: """ arch = self.clVars.Get('os_builder_arch_machine') local_arch = self.clVars.Get('os_arch_machine') # упрощенная проверка так как только 64 может собирать 32 if arch != local_arch: return Linux32(Chroot(chroot_path, cmd)) else: return Chroot(chroot_path, cmd) def emerge_ask(self, pretend, *params): """ Вывести информацию об обновлении """ deo = self.get_default_emerge_opts() param = [x for x in params if x.startswith("-")] packages = [x for x in params if not x.startswith("-")] chroot_path = self.clVars.Get('cl_builder_path') logfile = self._get_log_file() with EmergeParser(self.chrootize(chroot_path, EmergeCommand( list(packages), emerge_default_opts=deo, extra_params=param, logfile=logfile))) as emerge: try: emerge.question.action = lambda x: False emerge.run() if emerge.install_packages.list: emergelike = self.clVars.Get( 'update.cl_update_emergelist_set') == 'on' self._display_install_package(emerge, emergelike) if emerge.skipped_packages: self._display_error(emerge.skipped_packages) if not pretend: answer = self.askConfirm( _("Would you like to merge these packages?"), "yes") if answer == "no": emerge.command.send("no\n") raise KeyboardInterrupt else: return True else: self.printSUCCESS(_("Nothing to merge")) except EmergeError as e: #self.set_need_update(False) #self.emerge_cache.drop_cache("Emerge error") self._display_install_package(emerge, emergelike=True) self._display_error(emerge.prepare_error) raise self._startEmerging(emerge) return True def depclean(self): """ Выполнить очистку системы от лишних пакетов """ deo = self.get_default_emerge_opts() chroot_path = self.clVars.Get('cl_builder_path') logfile = self._get_log_file() with EmergeParser(self.chrootize(chroot_path, EmergeCommand( ["--depclean", "--ask=y"], logfile=logfile, emerge_default_opts=deo))) as emerge: try: emerge.question.action = lambda x: False emerge.run() if emerge.uninstall_packages.list: self._display_remove_list(emerge) if (self.askConfirm( _("Would you like to unmerge these unused packages " "(recommended)?")) != 'yes'): return True self._startEmerging(emerge) else: self.printSUCCESS(_("Nothing to unmerge")) except EmergeError: self._display_error(emerge.prepare_error) raise return True def chroot_command(self, builder_path, command, *params): """ Выполенине eix-update для репозиторием eix-update выполнятется только для тех репозиториев, которые обновлялись, если cl_update_eixsync_force==auto, либо все, если cl_update_eixupdate_force==force """ cmdpath = self.get_prog_path(command) if not cmdpath: return "skip" p = self.chroot_process(builder_path, cmdpath, *params, stderr=STDOUT) with writeFile(self._get_log_file()) as f: f.write(p.read()) if p.failed(): raise BuilderError(_("Failed to execute %s")%command) return True def update_task(self, task_name): """ Декоратор для добавления меток запуска и останова задачи """ def decor(f): def wrapper(*args, **kwargs): logger = EmergeLog(EmergeLogNamedTask(task_name), prefix=self.clVars.Get('cl_builder_path')) logger.mark_begin_task() ret = f(*args, **kwargs) if ret: logger.mark_end_task() return ret return wrapper return decor def rebuild_changed_packages(self, builder_path, repository_data): """ Пересобрать изменённые пакеты """ var_db_path = path.join(builder_path, 'var/db/pkg') map_rep = {k: pathJoin(builder_path, v) for k, v in repository_data} def rebuild_generator(): for pkg in InstalledPackageInfo.get_install_packages(var_db_path): try: if pkg['repository'] not in map_rep: yield pkg.atom elif pkg != EbuildInfo(pkg.atom, map_rep[pkg['repository']]): yield pkg.atom except EbuildInfoError: pass rebuild_list = map(lambda x: "=%s" % x, rebuild_generator()) if rebuild_list: return self.emerge_ask(False, "-1", *rebuild_list) return True class Driver(object): Package = None Id = None def __init__(self, builder_path="/"): self.builder_path = builder_path def generate(self): raise StopIteration def __iter__(self): yield (self.Id, '', self.Package) for x in self.generate(): yield x class NvidiaDriver(Driver): Id = 'nvidia-drivers' Package = 'x11-drivers/nvidia-drivers' Eclass = 'usr/portage/eclass/nvidia-driver.eclass' SkipVers = ('71', '96', '173', '304') def generate(self): nvidia_eclass = path.join(self.builder_path, self.Eclass) mask_prefix = "mask_" for mask in (x for x in readLinesFile(nvidia_eclass) if x.startswith(mask_prefix)): # пропустить сборку для 71, 96 и 173 if any(mask.startswith("%s%s" % (mask_prefix, x)) for x in self.SkipVers): continue mask = mask.partition('=')[2].strip("\n") yield (self.Id, mask, mask.replace('>=', '<')) class AtiDriver(Driver): Id = 'ati-drivers' Package = 'x11-drivers/ati-drivers' def pretend_package_install(self, atom, chroot_path, logfile=None): """ Получить список пакетов для установки :return: список пакетов """ if atom not in self.pretend_package_list: with EmergeParser(self.chrootize(chroot_path, EmergeCommand( [atom], extra_params=['-pv', '--ask=n'], logfile=logfile))) as emerge: emerge.question.default_answer = "n" emerge.run() self.pretend_package_list[atom] = list(emerge.install_packages.list) return self.pretend_package_list[atom] def _display_video_install_package(self, package_list, drv_name): """ Отобразить список устанавливаемых пакетов, если пакет не бинарный и не является непосредственно видеодрайвером - то он отмечен красной "*" :param package_list: список пакетов :param drv_name: имя драйвера (PN) :return: """ #asterisk = self.color_print.bold("*") #ebuild_wrong = TextState.Colors.RED ebuild_color = TextState.Colors.GREEN binary_color = TextState.Colors.PURPLE output_package_list = ", ".join( self.color_print.foreground(binary_color)(str(x)) if x['binary'] else self.color_print.foreground(ebuild_color)(str(x)) for x in package_list if x['PN'] != drv_name ) wrong_package = any(not x['binary'] and x['PN'] != drv_name for x in package_list) if wrong_package: self.printERROR(_("Depends %s")%output_package_list) else: self.printSUCCESS(_("Depends %s")%output_package_list) def fetch_drivers(self, builder_path, builder_distdir, builder_pkgdir): """ Скачать файлы для установки видеодрайверов :param builder_path: :param builder_distdir: :param builder_pkgdir: :return: """ distrdir_perm = (FilePermission.SetGid | FilePermission.UserAll | FilePermission.GroupRead | FilePermission.GroupExecute | FilePermission.OtherRead | FilePermission.OtherExecute) portage_group = 250 root_user = 0 logfile = self._get_log_file() driver_list = list(chain(self.NvidiaDriver(builder_path), self.AtiDriver(builder_path))) distrdir = path.join(builder_path, 'usr/portage/distfiles') pkgdir = path.join(builder_path, 'usr/portage/packages') for target_dn in (distrdir, pkgdir): makeDirectory(target_dn) chmod(target_dn, distrdir_perm) chown(target_dn, root_user, portage_group) pkgdir_files = [] distdir_files = [] repeat_driver_list = [] while driver_list or repeat_driver_list: drv_name, drv_mask, drv_atom = driver_list.pop(0) self.startTask(_("Calculating dependencies for %s") % drv_atom.strip('"').replace("<", "<")) package_list = self.pretend_package_install(drv_atom, builder_path, logfile=logfile) binary_map = {str(x): x['binary'] for x in package_list} self._display_video_install_package(package_list, drv_name) self.startTask(_("Fetching binary packages and sources tarballs") % [x for x in package_list if x['PN'] == drv_name][0]) ef = EmergeFetcher(self.chrootize(builder_path, EmergeCommand( ["=%s" % x for x in package_list], extra_params=["-Of", "--ask=n"], logfile="%s.2" % logfile))) try: for package in ef: pkg_name = str(package) if binary_map.get(pkg_name, False): for fn in package.files: pkgdir_files.append("%s/%s" % (package['CATEGORY'], fn)) else: for fn in package.files: distdir_files.append(fn) if ef.failed(): raise BuilderError(_("Failed to get %s") % drv_name) except EmergeFetcherError as e: if e.extension: self.printPre("\n%s\n"%e.extension) if e.errno == EmergeFetcherError.FetchErrno.NeedManually: raise BuilderError( _("Failed to fetch files for %s") % drv_name) repeat_driver_list.append([drv_name, drv_mask, drv_atom]) if not driver_list and repeat_driver_list: driver_list = repeat_driver_list repeat_driver_list = [] self.printWARNING(_("Waiting for unlock %s") % driver_list[0][0]) time.sleep(10) self.startTask(_("Cleaning and copying driver files")) for source_dn, source, target_dn, target in [ (builder_distdir, distdir_files, distrdir, find(distrdir, filetype=FindFileType.RegularFile, fullpath=False)), (builder_pkgdir, pkgdir_files, pkgdir, find(pkgdir, filetype=FindFileType.RegularFile, fullpath=False))]: # удаляем все ненужные файлы for fn in target: if fn not in source: removeFileWithEmptyDirectory(path.join(target_dn, fn), stopDirectory=target_dn) # копируем отсутствующие файлы for fn in source: if fn not in target: copyWithPath(path.join(source_dn, fn), target_dn, prefix=source_dn) return True def create_video_data(self, builder_path, repository_data): driver_list = list(chain(self.NvidiaDriver(builder_path), self.AtiDriver(builder_path))) logfile = self._get_log_file() cache_file = self.clVars.Get('builder.cl_builder_video_driver_path') map_rep = dict(repository_data) with writeFile(cache_file) as f: for drv_name, drv_mask, drv_atom in driver_list: package_list = self.pretend_package_install( drv_atom, builder_path, logfile=logfile) for package in package_list: if package['binary']: s = "{category} {pn} {pv} binary {drv} {mask}\n".format( category=package['CATEGORY'], pn=package['PN'], pv=package['PVR'], drv=drv_name, mask=drv_mask.strip('"')) f.write(s) else: if package['REPO'] not in map_rep: raise BuilderError( _("Failed to determine path " "for %s repository") % package['REPO']) s = "{category} {pn} {pv} {dn} {drv} {mask}\n".format( category=package['CATEGORY'], pn=package['PN'], pv=package['PV'], dn=map_rep[package['REPO']], drv=drv_name, mask=drv_mask.strip('"')) f.write(s) return True def remove_video_drivers(self, builder_path): """ Удалить данные и архивы для установки видео драйверов :param builder_path: путь до сборки :return: """ cache_file = self.clVars.Get('builder.cl_builder_video_driver_path') if path.exists(cache_file): removeFileWithEmptyDirectory(cache_file) distrdir = path.join(builder_path, 'usr/portage/distfiles') pkgdir = path.join(builder_path, 'usr/portage/packages') for target_dn, target in [ (distrdir, find(distrdir, fullpath=False)), (pkgdir, find(pkgdir, fullpath=False))]: # удаляем все найденные файлы for fn in target: removeFileWithEmptyDirectory(path.join(target_dn, fn), stopDirectory=target_dn) return True def reading_news(self, builder_path): """ 'Прочитать' новости :param builder_path: путь до сборки :return: """ eselect_command = "/usr/bin/eselect" p = self.chroot_process(builder_path, eselect_command, "--colour=yes", "news", "list", stderr=STDOUT) self.printPre(convert_console_to_xml( p.read()).replace(" ", "  ")) p = self.chroot_process(builder_path, eselect_command, "--colour=yes", "news", "read", "new", stderr=STDOUT) return p.success() def check_obsolete(self, builder_path): """ Проверка на устаревшие установленные пакеты :param builder_path: :return: """ chroot_eix = ChrootEix(builder_path, [], ChrootEix.Option.TestObsolete) l = chroot_eix.get_packages() if l: self.printERROR("Obsolete packages list:") mult = self.color_print.bold("*") for pkg in l: self.printDefault( " {mult} {package}".format( mult=mult, package=pkg['CATEGORY/PN'])) return True def emergelike(self, builder_path, cmd, *params): """ Запуск команды, которая подразумевает выполнение emerge """ cmd_path = self.get_prog_path(cmd) logfile = self._get_log_file() if not cmd_path: raise BuilderError(_("Failed to find the %s command") % cmd) with EmergeParser(self.chrootize(builder_path, CommandExecutor( cmd_path, params, logfile=logfile))) as emerge: self._startEmerging(emerge) return True def revdep_rebuild(self, builder_path, cmd, *params): """ Запуск revdep-rebulid """ cmd_path = self.get_prog_path(cmd) logfile = self._get_log_file() if not cmd_path: raise BuilderError(_("Failed to find the %s command") % cmd) with EmergeParser(self.chrootize(builder_path, CommandExecutor( builder_path, cmd_path, params, logfile=logfile))) as emerge: revdep = RevdepPercentBlock(emerge) self.addProgress() revdep.add_observer(self.setProgress) revdep.action = lambda x: ( self.endTask(), self.startTask(_("Assigning files to packages")) if "Assign" in revdep else None) self._startEmerging(emerge) return True def syncLaymanRepository(self, repname): """ Обновить репозиторий через layman """ chroot_path = self.clVars.Get('cl_builder_path') layman = self.get_prog_path('/usr/bin/layman') if not layman: raise BuilderError(_("The Layman tool is not found")) rpath = self.clVars.Select('cl_builder_other_rep_path', where='cl_builder_other_rep_name', eq=repname, limit=1) laymanname = path.basename(rpath) self.stash_cache(rpath, laymanname) try: if Git.is_git(rpath): self.addProgress() p = PercentProgress( "/usr/bin/chroot", chroot_path, layman, "-s", laymanname, part=1, atty=True) for perc in p.progress(): self.setProgress(perc) else: p = self.chroot_process( chroot_path, layman, "-s", repname, stderr=STDOUT) if p.failed(): raise BuilderError( _("Failed to update the {rname} repository").format( rname=repname), addon=p.read()) finally: self.unstash_cache(rpath, laymanname) return True def _update_binhost_packages(self): """ Выполнить команду обновления файла binhost (Packages.gz) :return: """ chroot_path = self.clVars.Get('cl_builder_path') os.system('/usr/bin/chroot %s /usr/sbin/emaint binhost -f &>/dev/null'% chroot_path) def cleanpkg(self): """ Очистка системы от устаревших distfiles и packages :return: """ builder_path = self.clVars.Get('cl_builder_path') portdirs = [ pathJoin(builder_path, x) for x in self.clVars.Get("builder.cl_builder_repository_location")] pkgfiles = get_packages_files_directory(*portdirs) distdirfiles = get_manifest_files_directory(*portdirs) distdir = self.clVars.Get('builder.cl_builder_linux_distdir') pkgdir = self.clVars.Get('builder.cl_builder_linux_pkgdir') logfile = self._get_log_file() logger = log("update_cleanpkg.log", filename=logfile, formatter="%(asctime)s - %(clean)s - %(message)s") return self._cleanpkg( distdir, pkgdir, distdirfiles, pkgfiles, logger) def raiseOutdate(self): """ Установить флаг данные о репозиториях устарели (необходим для выполнения eix-update и прочих команд обновляющих кэш :return: """ self.clVars.Set('cl_builder_outdate_set', 'on', force=True) def apply_branch_variables(self): """ Применить значение переменной для выбора веток репозиториев при обновлении """ self.clVars.Set('update.cl_update_branch_name', self.clVars.Get('builder.cl_builder_branch_name')) return True def isohybrid(self, image_file): """ Преобразовать ISO образ в гибридный :param image_file: iso образ :return: """ isohybrid = getProgPath("/usr/bin/isohybrid") if not isohybrid: raise BuilderError(_("Isohybrid utility not found")) if not path.exists(image_file): raise BuilderError(_("Image not found")) if self.clVars.Get('os_builder_arch_machine') == 'x86_64': cmd = [isohybrid, "--uefi", image_file] else: cmd = [isohybrid, image_file] isohybrid_process = process(*cmd) return isohybrid_process.success() def _list_file(self, iso_file): """ .list файл по iso файлу :param iso_file: :return: """ if iso_file.endswith(".iso"): return "%s.list" % iso_file[:-4] else: return "%s.list" % iso_file def _digest_file(self, iso_file): return "%s.DIGEST" % iso_file def create_package_list(self, chroot, iso_file): """ Создает список установленных пакетов в chroot директории и сохраняет в iso_file :return: """ pkgdir = path.join(chroot, 'var/db/pkg') list_file = self._list_file(iso_file) with writeFile(list_file) as f: f.write("\n".join(sorted( x.atom for x in InstalledPackageInfo.get_install_packages( pkg_dir=pkgdir)))) return True def create_digest(self, isofile): """ Создать контрольную сумму для файла """ template = """# %(alg)s HASH\n%(digest)s %(filename)s\n""" digestfile = self._digest_file(isofile) try: with writeFile(digestfile) as f: f.writelines(map(lambda x: template % { 'alg': x.upper(), 'digest': process("%ssum" % x, isofile).read().partition(' ')[0], 'filename': path.basename(isofile)}, ["md5", "sha1"])) except (IOError, OSError): return False return True @property def chroot_process(self): chroot_cmd = getProgPath("/usr/bin/chroot") if not chroot_cmd: raise BuilderError(_("Chroot command not found")) arch = self.clVars.Get('builder.os_builder_arch_machine') local_arch = self.clVars.Get('os_arch_machine') process_func = process if arch != local_arch: process_func = partial(process_func, "/usr/bin/linux32") return partial(process_func, chroot_cmd) def get_default_emerge_opts(self): dv = self.clVars.Get('builder.cl_builder_linux_datavars') if dv: deo = dv.Get('cl_emerge_default_opts') else: deo = super(Builder, self).get_default_emerge_opts() return deo def emerge(self, builder_path, use, *params): """ Выполнить сборку пакета """ deo = self.get_default_emerge_opts() extra_params = [x for x in params if x.startswith("-")] packages = [x for x in params if not x.startswith("-")] with EmergeParser(self.chrootize(builder_path, EmergeCommand( packages, emerge_default_opts=deo, extra_params=extra_params, use=use))) as emerge: try: emerge.question.action = lambda x: False emerge.run() if not emerge.install_packages.list: return True except EmergeError: self.emerge_cache.drop_cache("Emerge error") self._display_error(emerge.prepare_error) raise self._startEmerging(emerge) return True def remove_list_digest(self, isofile): for fn in (self._digest_file(isofile), self._list_file(isofile)): if path.exists(fn): os.unlink(fn) return True def recount_files(self, builder_path, exclude_files): """ Посчитать файлы в собираемой системе и исключить exclude файлы :param builder_path: :param exclude_files: :return: """ builder_path = builder_path.encode('utf-8') all_count = countFiles(builder_path) exclude_sum = sum(countFiles(path.join(builder_path, x)) for x in exclude_files) self.clVars.Set('os_builder_linux_files', str(all_count - exclude_sum), force=True) return True def create_dev_nodes(self,directory): """Create nodes for dev http://dev.gentoo.org/~a3li/openrc.txt""" devPath = path.join(directory, "dev") # clean dev for pathname, dirs, files in os.walk(devPath, topdown=False): map(lambda x: os.unlink(path.join(pathname, x)), files) map(lambda x: os.unlink(x) if path.islink(x) else os.rmdir(x), map(lambda x: path.join(pathname, x), dirs)) for node, mode, dmode, major, minor in [ ("console", 0600, stat.S_IFCHR, 5, 1), ("tty1", 0600, stat.S_IFCHR, 4, 1), ("null", 0666, stat.S_IFCHR, 1, 3), ("zero", 0666, stat.S_IFCHR, 1, 5)]: nodePath = path.join(devPath, node) os.mknod(nodePath, mode | dmode, os.makedev(major, minor)) os.chmod(nodePath, mode) return True def check_build_run(self): """ Проверить повторный запуск """ build_id = self.clVars.Get('cl_builder_id') names = self.Method.All pid = os.getpid() filter_func = lambda x: ('id' in x and x['id'] == build_id and x['name'] in names and x['os_pid'] != pid) dv = self.clVars if any(search_worked_process2(dv, filter_func=filter_func)): raise BuilderError(_("Builder action for %s is already running. " "Try to run later.")%build_id) return True def update_menu(self, dn): with IsoDistributive(dn) as iso: d = iso.getDirectory() root_path = path.relpath(dn, d) self.clVars.Set('cl_builder_iso_path', dn, force=True) self.clVars.Set('cl_builder_target', iso, force=True) self.clVars.Set('cl_builder_path', d, force=True) self.clVars.getInfo('cl_builder_videodrv_set').autodetect = True self.clVars.Invalidate('cl_builder_videodrv_set', force=True) from calculate.lib.cl_template import templateFunction templateFunction.installProg = {} templateFunction.installCategory = [] self.applyTemplates(d, False, False, root_path) return True def remove_flash_tmp(self): try: image = self.clVars.Get('cl_builder_image') if image: image.close() except DistributiveError: pass flashTmp = path.join(self.clVars.Get('cl_builder_flash_path'), "tmp") if path.exists(flashTmp) and not listDirectory(flashTmp): try: os.rmdir(flashTmp) except (OSError,IOError) as e: self.printWARNING(str(e)) return True def remount_rw(self, dn): """ Перемонтировать каталог для чтения/записи (используется для flash) :param dn: каталог :return: """ if not dn: raise BuilderError(_("Failed to remount Flash drive")) p = process('/bin/mount', '-o', 'remount,rw', dn, stderr=STDOUT) if p.failed(): self.printERROR(p.read().strip()) return False return True def sync_vmlinuz(self, flash_dn): """ Извлечение ядер из iso образов :return: """ boot_dn = path.join(flash_dn, "boot") for fn in listDirectory(boot_dn, fullPath=True): if fn not in ("vmlinuz", "initrd"): try: if not path.isdir(fn): os.unlink(fn) except (IOError, OSError) as e: self.printERROR(str(e)) raise BuilderError( _("Failed to clean /boot directory on Flash drive")) for data in self.clVars.Get('cl_builder_image_data'): isofn = data[2] vmlinuz_orig = data[3] vmlinuz = data[4] initrd_orig = data[5] initrd = data[6] with IsoDistributive(isofn) as iso: dn = iso.getDirectory() vmlinuz_orig = path.join(dn, "boot", vmlinuz_orig) initrd_orig = path.join(dn, "boot", initrd_orig) vmlinuz = path.join(boot_dn, vmlinuz) initrd = path.join(boot_dn, initrd) try: open(vmlinuz, 'w').write(open(vmlinuz_orig, 'rb').read()) open(initrd, 'w').write(open(initrd_orig, 'rb').read()) except (IOError, OSError) as e: self.printERROR(str(e)) raise BuilderError(_("Failed to extract kernel from %s") % isofn) return True