calculate-utils-3-update/pym/builder/builder.py

# -*- coding: utf-8 -*-

# Copyright 2015-2016 Mir Calculate. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from itertools import chain

import re
import sys
import time
import stat
from calculate.core.server.gen_pid import search_worked_process2
from calculate.core.setup_package import ChainProgressTemplate
from calculate.lib.cl_template import templateFunction
from calculate.lib.datavars import DataVars, Variable
from calculate.lib.utils.colortext import (TextState, get_color_print,
                                           convert_console_to_xml)
from calculate.lib.utils.common import CmdlineParams
from calculate.lib.utils.files import (
    pathJoin, PercentProgress, getProgPath, process, STDOUT, removeDir,
    makeDirectory, writeFile, readLinesFile, chmod, chown, FilePermission,
    find, FindFileType, removeFileWithEmptyDirectory, check_rw,
    copyWithPath, countFiles, listDirectory, getRunCommands, isMount, readFile)
from calculate.lib.utils.git import Git
from calculate.lib.utils.portage import (Layman, EmergeLog, EmergeLogNamedTask,
                                         InstalledPackageInfo, EbuildInfoError,
                                         EbuildInfo, ChrootEix,
                                         get_packages_files_directory,
                                         get_manifest_files_directory)
from calculate.update.emerge_parser import (EmergeParser,
                                            EmergeError, EmergeCommand, Chroot,
                                            Linux32, CommandExecutor,
                                            RevdepPercentBlock, EmergeCache)
from calculate.lib.cl_log import log
from .build_storage import Build
from calculate.update.update import Update
from calculate.install.distr import (Distributive, IsoDistributive,
                                     DistributiveError)
from functools import partial
import os
from os import path
from .datavars import BuilderError
from .emerge_fetch import EmergeFetcher, EmergeFetcherError
from calculate.lib.utils.grub import GrubCommand

_ = lambda x: x
from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate)

setLocalTranslate('cl_builder3', sys.modules[__name__])
__ = getLazyLocalTranslate(_)


class Builder(Update):
    """Основной объект для выполнения действий связанных со сборкой системы

    """

    class Method(object):
        Prepare = "builder_prepare"
        Break = "builder_break"
        Update = "builder_update"
        Restore = "builder_restore"
        Image = "builder_image"
        Profile = "builder_profile"
        UpdateMenu = "update_livemenu"
        All = (Prepare, Break, Update, Restore, Image, Profile)

    def init(self):
        self.pretend_package_list = {}
        self.update_map = {}
        self.color_print = get_color_print()
        self.emerge_cache = EmergeCache()

    def mount_target(self, target):
        dir_distro = target.convertToDirectory()
        dir_distro.mountSystemDirectories(skip=())
        return True

    def umount_system(self, target):
        dir_distro = target.convertToDirectory()
        dir_distro.umountSystemDirectories()
        return True

    def detach_target(self, target):
        """
        @type target: Distributive
        """
        if target:
            target.reserve()
        return True

    def close_build(self, build, clear=False, clear_pkg=False):
        """
        @type build:Build
        """
        if build:
            builder_path = self.clVars.Get('cl_builder_path')
            if builder_path:
                if isMount(builder_path):
                    build.close_distributive()
            build.remove()
            if builder_path:
                if clear:
                    build.distributive.post_clear()
                pkgdir = self.clVars.Get('cl_builder_pkgdir')
                if clear_pkg:
                    if '/remote/' in pkgdir:
                        if path.exists(pkgdir):
                            removeDir(pkgdir)
                            Distributive.clear_empty_directories(pkgdir)
        return True

    def restore_build(self, build):
        if build:
            build.restore()
        return True

    def save_build(self, build, dv):
        build.pkgdir = self.clVars.Get('cl_builder_pkgdir')
        if dv:
            overlays = dv.Get('cl_update_rep_name')
            build.set_overlays(overlays)
        build.save()
        return True

    def set_builder_action(self, action_name):
        self.clVars.Set('cl_builder_action', action_name, force=True)
        return True

    def prepare_iso(self, dn):
        self.endTask()
        self.startTask(_("Prepare ISO data"))
        root_path = path.relpath(dn, self.clVars.Get('cl_builder_path'))
        self.applyTemplates(self.clVars.Get('cl_builder_target'),
                            False, False, root_path)
        self.endTask()
        self.startTask(_("Pack ISO image"))
        self.addProgress()

    def prepare_image(self, image):
        image.eventPrepareIso.connect(self.prepare_iso)
        return True

    def remove_repositories(self, repname):
        dv = self.clVars.Get('cl_builder_linux_datavars')
        rpath = dv.Select("cl_update_rep_path",
                          where="cl_update_rep_name", eq=repname, limit=1)
        chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
        rpath_orig = rpath[len(chroot_path):]
        layman = Layman(self.clVars.Get('update.cl_update_layman_installed'),
                        self.clVars.Get('update.cl_update_layman_make'),
                        self.clVars.Get('update.cl_update_layman_conf'),
                        prefix=self.clVars.Get('cl_builder_path'))
        if repname not in ("gentoo", "portage"):
            layman.remove(repname, rpath_orig)
            removeDir(rpath)
        return True

    def set_profile(self, profile_shortname, chroot_path):
        profile = self.clVars.Select('cl_builder_profile_path',
                                     where='cl_builder_profile_shortname',
                                     eq=profile_shortname, limit=1)
        if not profile:
            raise BuilderError(_("Failed to determine profile %s") %
                               profile_shortname)
        chroot_join = lambda x: path.join(chroot_path, x)
        profile_path = path.relpath(profile, chroot_join('etc/portage'))
        try:
            for rm_fn in filter(path.lexists,
                                (chroot_join('etc/make.profile'),
                                 chroot_join('etc/portage/make.profile'))):
                os.unlink(rm_fn)
            os.symlink(profile_path, chroot_join('etc/portage/make.profile'))
        except (OSError, IOError) as e:
            raise BuilderError(_("Failed to set the profile: %s") % str(e))
        return True

    def apply_templates(self, target=None, useClt=None, cltFilter=False,
                        root=None,
                        useDispatch=True, action="merge", distro_dv=None):
        """
        Применить шаблоны.

        Args:
        target: дистрибутив, куда необходимо выполнить шаблоны (/ по умолчанию)
        useClt: использовать clt шаблоны
        cltFilter: применять фильтр на clt шаблоны
        root: каталог, куда будут наложны шаблоны (cl_root_path)
        """
        from calculate.lib.cl_template import (TemplatesError,
                                               ProgressTemplate,
                                               templateFunction)

        templateFunction.installProg = {}
        templateFunction.installCategory = []
        if target is None:
            chroot = '/'
        elif isinstance(target, Distributive):
            chroot = target.getDirectory()
        else:
            chroot = target
        if root is None:
            root = '/'
        elif isinstance(root, Distributive):
            root = root.getDirectory()
        clVars = DataVars()
        clTempl = None
        try:
            clVars.importData()
            clVars.Set('os_arch_machine',
                       self.clVars.Get('builder.os_builder_arch_machine'),
                       force=True)
            if distro_dv:
                clVars.Set('cl_template_path',
                           [pathJoin(chroot, x)
                            for x in distro_dv.Get('cl_template_path')],
                           force=True)
                clVars.Set('cl_template_path_use',
                           clVars.Get('cl_template_path'), force=True)
            clVars.Set('cl_env_path',
                       [pathJoin(chroot, x) for x in clVars.Get('cl_env_path')],
                       force=True)
            clVars.Set('cl_make_profile', path.join(chroot,
                                                    'etc/portage/make.profile'),
                       force=True)
            clVars.Set('cl_action', action, force=True)
            clVars.Set('cl_chroot_status', 'on', force=True)
            for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
                            "update.cl_update_world"):
                clVars.Set(copyvar, self.clVars.Get(copyvar), force=True)

            clVars.iniCache = {}
            clVars.flIniFile()
            cltFilter = True if cltFilter in (True, "on") else False
            clVars.Set("cl_chroot_path", chroot, True)
            clVars.Set("cl_root_path", root, True)
            # определение каталогов содержащих шаблоны
            dirs_list, files_list = ([], [])
            useClt = useClt in ("on", True)
            self.addProgress()
            nullProgress = lambda *args, **kw: None
            dispatch = self.dispatchConf if useDispatch else None

            clTempl = ProgressTemplate(nullProgress, clVars,
                                       cltObj=useClt,
                                       cltFilter=cltFilter,
                                       printSUCCESS=self.printSUCCESS,
                                       printWARNING=self.printWARNING,
                                       askConfirm=self.askConfirm,
                                       dispatchConf=dispatch,
                                       printERROR=self.printERROR)

            def execute_command(cmd, lang):
                chroot_path = self.clVars.Get('cl_builder_path')
                env = dict(os.environ)
                env['TERM'] = "linux"
                env['EINFO_QUIET'] = "yes"
                return self.chroot_process(
                    chroot_path, cmd, lang=lang, envdict=dict(os.environ))

            # замена выполения команд: вместо оычного запуска - запуск через
            # /usr/bin/chroot
            clTempl.execute_command = execute_command
            clTempl.applyTemplates()
            if clTempl.hasError():
                if clTempl.getError():
                    raise TemplatesError(clTempl.getError())
        finally:
            clVars.close()
            if clTempl:
                if clTempl.cltObj:
                    clTempl.cltObj.closeFiles()
                clTempl.closeFiles()
        return True

    def get_prog_path(self, progname):
        chroot_path = self.clVars.Get('builder.cl_builder_path')
        return getProgPath(progname, chroot_path)

    def _eixUpdateCommand(self, eix_cmd, countRep):
        chroot_path = self.clVars.Get('cl_builder_path')
        return PercentProgress("/usr/bin/chroot", chroot_path, eix_cmd,
                               "-F", part=countRep or 1, atty=True)

    def regenCache(self, repname):
        with self.clVars.useDefaultModule("update"):
            return super(Builder, self).regenCache(repname)

    def syncRepositories(self, repname, clean_on_error=True):
        with self.clVars.useDefaultModule("update"):
            return super(Builder, self).syncRepositories(
                repname, clean_on_error=clean_on_error)

    def _regenCache_process(self, progname, repname, cpu_num):
        chroot_path = self.clVars.Get('builder.cl_builder_path')
        return self.chroot_process(chroot_path,
                                   progname, "--repo=%s" % repname, "--update",
                                   "--jobs=%s" % cpu_num, stderr=STDOUT)

    def clear_log(self, builder_id_path):
        logname = "build-%s" % builder_id_path
        mainlog = self.clVars.Get('core.cl_log_path')
        logpath = path.join(mainlog, logname)
        if path.exists(logpath):
            removeDir(logpath)
        makeDirectory(logpath)
        return True

    def _get_log_file(self):
        logname = "build-%s/%s" % (self.clVars.Get('cl_builder_id_path'),
                                   self.clVars.Get('cl_task_name'))
        mainlog = self.clVars.Get('core.cl_log_path')
        return path.join(mainlog, logname)

    def chrootize(self, chroot_path, cmd):
        """
        :param chroot_path:
        :param cmd:
        :return:
        """
        arch = self.clVars.Get('os_builder_arch_machine')
        local_arch = self.clVars.Get('os_arch_machine')
        # упрощенная проверка так как только 64 может собирать 32
        if arch != local_arch:
            return Linux32(Chroot(chroot_path, cmd))
        else:
            return Chroot(chroot_path, cmd)

    def emerge_ask(self, pretend, *params):
        """
        Вывести информацию об обновлении
        """
        deo = self.get_default_emerge_opts()
        param = [x for x in params if x.startswith("-")]
        packages = [x for x in params if not x.startswith("-")]
        chroot_path = self.clVars.Get('cl_builder_path')
        logfile = self._get_log_file()
        with EmergeParser(self.chrootize(chroot_path, EmergeCommand(
                list(packages),
                emerge_default_opts=deo,
                extra_params=param,
                logfile=logfile))) as emerge:
            try:
                emerge.question.action = lambda x: False
                emerge.run()
                if emerge.install_packages.list:
                    emergelike = self.clVars.Get(
                        'update.cl_update_emergelist_set') == 'on'
                    self._display_install_package(emerge, emergelike)
                    if emerge.skipped_packages:
                        self._display_error(emerge.skipped_packages)
                    if not pretend:
                        answer = self.askConfirm(
                            _("Would you like to merge these packages?"), "yes")
                        if answer == "no":
                            emerge.command.send("no\n")
                            raise KeyboardInterrupt
                    else:
                        return True
                else:
                    self.printSUCCESS(_("Nothing to merge"))
            except EmergeError:
                # self.set_need_update(False)
                # self.emerge_cache.drop_cache("Emerge error")
                self._display_install_package(emerge, emergelike=True)
                self._display_error(emerge.prepare_error)
                raise
            self._startEmerging(emerge)
        return True

    def depclean(self):
        """
        Выполнить очистку системы от лишних пакетов
        """
        deo = self.get_default_emerge_opts()
        chroot_path = self.clVars.Get('cl_builder_path')
        logfile = self._get_log_file()
        with EmergeParser(self.chrootize(chroot_path, EmergeCommand(
                ["--depclean", "--ask=y"],
                logfile=logfile,
                emerge_default_opts=deo))) as emerge:
            try:
                emerge.question.action = lambda x: False
                emerge.run()
                if emerge.uninstall_packages.list:
                    self._display_remove_list(emerge)
                    if (self.askConfirm(
                            _("Would you like to unmerge these unused packages "
                              "(recommended)?")) != 'yes'):
                        return True
                    self._startEmerging(emerge)
                else:
                    self.printSUCCESS(_("Nothing to unmerge"))
            except EmergeError:
                self._display_error(emerge.prepare_error)
                raise
        return True

    def chroot_command(self, builder_path, command, *params):
        """
        Выполенине eix-update для репозиторием

        eix-update выполнятется только для тех репозиториев, которые
        обновлялись, если cl_update_eixsync_force==auto, либо
        все, если cl_update_eixupdate_force==force
        """
        cmdpath = self.get_prog_path(command)

        if not cmdpath:
            return "skip"
        p = self.chroot_process(builder_path, cmdpath, *params, stderr=STDOUT)
        with writeFile(self._get_log_file()) as f:
            f.write(p.read())
        if p.failed():
            raise BuilderError(_("Failed to execute %s") % command)
        return True

    def update_task(self, task_name):
        """
        Декоратор для добавления меток запуска и останова задачи
        """

        def decor(f):
            def wrapper(*args, **kwargs):
                logger = EmergeLog(EmergeLogNamedTask(task_name),
                                   prefix=self.clVars.Get('cl_builder_path'))
                logger.mark_begin_task()
                ret = f(*args, **kwargs)
                if ret:
                    logger.mark_end_task()
                return ret

            return wrapper

        return decor

    def rebuild_changed_packages(self, builder_path, repository_data):
        """
        Пересобрать изменённые пакеты
        """
        var_db_path = path.join(builder_path, 'var/db/pkg')
        map_rep = {k: pathJoin(builder_path, v)
                   for k, v in repository_data}

        def rebuild_generator():
            for pkg in InstalledPackageInfo.get_install_packages(var_db_path):
                try:
                    if pkg['repository'] not in map_rep:
                        yield pkg.atom
                    elif pkg != EbuildInfo(pkg.atom,
                                           map_rep[pkg['repository']]):
                        yield pkg.atom
                except EbuildInfoError:
                    pass

        rebuild_list = map(lambda x: "=%s" % x, rebuild_generator())
        if rebuild_list:
            return self.emerge_ask(False, "-1", *rebuild_list)
        return True

    class Driver(object):
        Package = None
        Id = None

        def __init__(self, builder_path="/"):
            self.builder_path = builder_path

        def generate(self):
            raise StopIteration

        def __iter__(self):
            yield (self.Id, '', self.Package)
            for x in self.generate():
                yield x

    class NvidiaDriver(Driver):
        Id = 'nvidia-drivers'
        Package = 'x11-drivers/nvidia-drivers'
        Eclass = 'usr/portage/eclass/nvidia-driver.eclass'
        SkipVers = ('71', '96', '173', '304')

        def generate(self):
            nvidia_eclass = path.join(self.builder_path, self.Eclass)
            mask_prefix = "mask_"
            for mask in (x for x in readLinesFile(nvidia_eclass)
                         if x.startswith(mask_prefix)):
                # пропустить сборку для 71, 96 и 173
                if any(mask.startswith("%s%s" % (mask_prefix, x))
                       for x in self.SkipVers):
                    continue
                mask = mask.partition('=')[2].strip("\n")
                yield (self.Id, mask, mask.replace('>=', '<'))

    class AtiDriver(Driver):
        Id = 'ati-drivers'
        Package = 'x11-drivers/ati-drivers'

    def pretend_package_install(self, atom, chroot_path, logfile=None):
        """
        Получить список пакетов для установки
        :return: список пакетов
        """
        deo = self.get_default_emerge_opts()
        if atom not in self.pretend_package_list:
            with EmergeParser(self.chrootize(chroot_path, EmergeCommand(
                    [atom], extra_params=['-pv', '--ask=n'],
                    emerge_default_opts=deo,
                    logfile=logfile))) as emerge:
                emerge.question.default_answer = "n"
                emerge.run()
                self.pretend_package_list[atom] = list(
                    emerge.install_packages.list)
        return self.pretend_package_list[atom]

    def _display_video_install_package(self, package_list, drv_name):
        """
        Отобразить список устанавливаемых пакетов, если пакет не бинарный
        и не является непосредственно видеодрайвером - то он отмечен красной
        "*"
        :param package_list: список пакетов
        :param drv_name: имя драйвера (PN)
        :return:
        """
        # asterisk = self.color_print.bold("*")
        # ebuild_wrong = TextState.Colors.RED
        ebuild_color = TextState.Colors.GREEN
        binary_color = TextState.Colors.PURPLE
        output_package_list = ", ".join(
            self.color_print.foreground(binary_color)(str(x))
            if x['binary'] else
            self.color_print.foreground(ebuild_color)(str(x))
            for x in package_list if x['PN'] != drv_name
        )
        wrong_package = any(not x['binary'] and x['PN'] != drv_name
                            for x in package_list)
        if wrong_package:
            self.printERROR(_("Depends %s") % output_package_list)
        else:
            self.printSUCCESS(_("Depends %s") % output_package_list)

    def fetch_drivers(self, builder_path, builder_distdir, builder_pkgdir):
        """
        Скачать файлы для установки видеодрайверов
        :param builder_path:
        :param builder_distdir:
        :param builder_pkgdir:
        :return:
        """
        distrdir_perm = (FilePermission.SetGid |
                         FilePermission.UserAll |
                         FilePermission.GroupRead |
                         FilePermission.GroupExecute |
                         FilePermission.OtherRead |
                         FilePermission.OtherExecute)
        portage_group = 250
        root_user = 0
        logfile = self._get_log_file()
        deo = self.get_default_emerge_opts()

        driver_list = list(chain(self.NvidiaDriver(builder_path),
                                 self.AtiDriver(builder_path)))
        distrdir = path.join(builder_path, 'usr/portage/distfiles')
        pkgdir = path.join(builder_path, 'usr/portage/packages')
        for target_dn in (distrdir, pkgdir):
            makeDirectory(target_dn)
            chmod(target_dn, distrdir_perm)
            chown(target_dn, root_user, portage_group)

        pkgdir_files = []
        distdir_files = []
        repeat_driver_list = []
        while driver_list or repeat_driver_list:
            drv_name, drv_mask, drv_atom = driver_list.pop(0)
            self.startTask(_("Calculating dependencies for %s") %
                           drv_atom.strip('"').replace("<", "&lt;"))
            package_list = self.pretend_package_install(drv_atom, builder_path,
                                                        logfile=logfile)

            binary_map = {str(x): x['binary'] for x in package_list}

            self._display_video_install_package(package_list, drv_name)

            self.startTask(_("Fetching binary packages and sources tarballs") %
                           [x for x in package_list if x['PN'] == drv_name][0])

            ef = EmergeFetcher(self.chrootize(builder_path, EmergeCommand(
                ["=%s" % x for x in package_list], emerge_default_opts=deo,
                extra_params=["-Of", "--ask=n"], logfile="%s.2" % logfile)))
            try:
                for package in ef:
                    pkg_name = str(package)
                    if binary_map.get(pkg_name, False):
                        for fn in package.files:
                            pkgdir_files.append("%s/%s" % (package['CATEGORY'],
                                                           fn))
                    else:
                        for fn in package.files:
                            distdir_files.append(fn)
                    if ef.failed():
                        raise BuilderError(_("Failed to get %s") % drv_name)
            except EmergeFetcherError as e:
                if e.extension:
                    self.printPre("\n%s\n" % e.extension)
                if e.errno == EmergeFetcherError.FetchErrno.NeedManually:
                    raise BuilderError(
                        _("Failed to fetch files for %s") % drv_name)
                repeat_driver_list.append([drv_name, drv_mask, drv_atom])

            if not driver_list and repeat_driver_list:
                driver_list = repeat_driver_list
                repeat_driver_list = []
                self.printWARNING(_("Waiting for unlock %s")
                                  % driver_list[0][0])
                time.sleep(10)

        self.startTask(_("Cleaning and copying driver files"))
        for source_dn, source, target_dn, target in [
            (builder_distdir, distdir_files,
             distrdir, find(distrdir,
                            filetype=FindFileType.RegularFile,
                            fullpath=False)),
            (builder_pkgdir, pkgdir_files,
             pkgdir, find(pkgdir,
                          filetype=FindFileType.RegularFile,
                          fullpath=False))]:
            # удаляем все ненужные файлы
            for fn in target:
                if fn not in source:
                    removeFileWithEmptyDirectory(path.join(target_dn, fn),
                                                 stopDirectory=target_dn)
            # копируем отсутствующие файлы
            for fn in source:
                if fn not in target:
                    copyWithPath(path.join(source_dn, fn),
                                 target_dn, prefix=source_dn)

        return True

    def create_video_data(self, builder_path, repository_data):
        driver_list = list(chain(self.NvidiaDriver(builder_path),
                                 self.AtiDriver(builder_path)))
        logfile = self._get_log_file()
        cache_file = self.clVars.Get('builder.cl_builder_video_driver_path')
        map_rep = dict(repository_data)
        video_ebuilds = []
        with writeFile(cache_file) as f:
            for drv_name, drv_mask, drv_atom in driver_list:
                package_list = self.pretend_package_install(
                    drv_atom, builder_path, logfile=logfile)
                for package in package_list:
                    if package['binary']:
                        s = "{category} {pn} {pv} binary {drv} {mask}\n".format(
                            category=package['CATEGORY'],
                            pn=package['PN'],
                            pv=package['PVR'],
                            drv=drv_name,
                            mask=drv_mask.strip('"'))
                        f.write(s)
                    else:
                        if package['REPO'] not in map_rep:
                            raise BuilderError(
                                _("Failed to determine path "
                                  "for %s repository") % package['REPO'])
                        s = "{category} {pn} {pv} {dn} {drv} {mask}\n".format(
                            category=package['CATEGORY'],
                            pn=package['PN'],
                            pv=package['PVR'],
                            dn=map_rep[package['REPO']],
                            drv=drv_name,
                            mask=drv_mask.strip('"'))
                        video_ebuilds.append(
                            "%s/%s/%s/%s-%s.ebuild"%(map_rep[package['REPO']],
                                                  package['CATEGORY'],
                                                  package['PN'],
                                                  package['PN'],
                                                  package['PVR']))
                        f.write(s)
        self.clVars.Set('cl_builder_video_ebuilds', video_ebuilds)
        return True

    def remove_video_drivers(self, builder_path):
        """
        Удалить данные и архивы для установки видео драйверов
        :param builder_path: путь до сборки
        :return:
        """
        cache_file = self.clVars.Get('builder.cl_builder_video_driver_path')
        if path.exists(cache_file):
            removeFileWithEmptyDirectory(cache_file)
        distrdir = path.join(builder_path, 'usr/portage/distfiles')
        pkgdir = path.join(builder_path, 'usr/portage/packages')
        for target_dn, target in [
            (distrdir, find(distrdir,
                            fullpath=False)),
            (pkgdir, find(pkgdir,
                          fullpath=False))]:
            # удаляем все найденные файлы
            for fn in target:
                removeFileWithEmptyDirectory(path.join(target_dn, fn),
                                             stopDirectory=target_dn)
        return True

    def reading_news(self, builder_path):
        """
        'Прочитать' новости
        :param builder_path: путь до сборки
        :return:
        """
        eselect_command = "/usr/bin/eselect"
        p = self.chroot_process(builder_path, eselect_command, "--colour=yes",
                                "news", "list", stderr=STDOUT)
        self.printPre(convert_console_to_xml(
            p.read()).replace("  ", "&nbsp;&nbsp;"))
        p = self.chroot_process(builder_path, eselect_command, "--colour=yes",
                                "news", "read", "new", stderr=STDOUT)
        return p.success()

    def check_obsolete(self, builder_path):
        """
        Проверка на устаревшие установленные пакеты
        :param builder_path:
        :return:
        """
        chroot_eix = ChrootEix(builder_path, [], ChrootEix.Option.TestObsolete)
        l = chroot_eix.get_packages()
        if l:
            self.printERROR(_("Obsolete packages list:"))
            mult = self.color_print.bold("*")
            for pkg in l:
                self.printDefault(
                    "&nbsp;{mult} {package}".format(
                        mult=mult, package=pkg['CATEGORY/PN']))
        return True

    def chroot_emergelike(self, builder_path, cmd, *params):
        """
        Запуск команды, которая подразумевает выполнение emerge
        """
        cmd_path = self.get_prog_path(cmd)
        logfile = self._get_log_file()
        if not cmd_path:
            raise BuilderError(_("Failed to find the %s command") % cmd)
        with EmergeParser(self.chrootize(builder_path, CommandExecutor(
                cmd_path, params,
                logfile=logfile))) as emerge:
            self._startEmerging(emerge)
        return True

    def chroot_revdep_rebuild(self, builder_path, cmd, *params):
        """
        Запуск revdep-rebulid
        """
        cmd_path = self.get_prog_path(cmd)
        logfile = self._get_log_file()
        if not cmd_path:
            raise BuilderError(_("Failed to find the %s command") % cmd)
        with EmergeParser(self.chrootize(builder_path, CommandExecutor(
                cmd_path, params, logfile=logfile))) as emerge:
            revdep = RevdepPercentBlock(emerge)
            self.addProgress()
            revdep.add_observer(self.setProgress)
            revdep.action = lambda x: (
                self.endTask(), self.startTask(_("Assigning files to packages"))
                if "Assign" in revdep else None)
            self._startEmerging(emerge)
        return True

    def syncLaymanRepository(self, repname):
        """
        Обновить репозиторий через layman
        """
        chroot_path = self.clVars.Get('cl_builder_path')
        layman = self.get_prog_path('/usr/bin/layman')
        if not layman:
            raise BuilderError(_("The Layman tool is not found"))

        rpath = self.clVars.Select('cl_builder_other_rep_path',
                                   where='cl_builder_other_rep_name',
                                   eq=repname, limit=1)
        laymanname = path.basename(rpath)
        self.stash_cache(rpath, laymanname)
        try:
            if Git.is_git(rpath):
                self.addProgress()
                p = PercentProgress(
                    "/usr/bin/chroot", chroot_path,
                    layman, "-s", laymanname, part=1, atty=True)
                for perc in p.progress():
                    self.setProgress(perc)
            else:
                p = self.chroot_process(
                    chroot_path, layman, "-s", repname, stderr=STDOUT)
            if p.failed():
                raise BuilderError(
                    _("Failed to update the {rname} repository").format(
                        rname=repname),
                    addon=p.read())
        finally:
            self.unstash_cache(rpath, laymanname)
        return True

    def _update_binhost_packages(self):
        """
        Выполнить команду обновления файла binhost (Packages.gz)
        :return:
        """
        chroot_path = self.clVars.Get('cl_builder_path')
        os.system('/usr/bin/chroot %s /usr/sbin/emaint binhost -f &>/dev/null' %
                  chroot_path)

    def cleanpkg(self):
        """
        Очистка системы от устаревших distfiles и packages
        :return:
        """
        builder_path = self.clVars.Get('cl_builder_path')
        portdirs = [
            pathJoin(builder_path, x)
            for x in self.clVars.Get("builder.cl_builder_repository_location")]
        pkgfiles = get_packages_files_directory(*portdirs)
        distdirfiles = get_manifest_files_directory(*portdirs)
        distdir = self.clVars.Get('builder.cl_builder_linux_distdir')
        pkgdir = self.clVars.Get('builder.cl_builder_linux_pkgdir')
        logfile = self._get_log_file()

        logger = log("update_cleanpkg.log", filename=logfile,
                     formatter="%(asctime)s - %(clean)s - %(message)s")

        return self._cleanpkg(
            distdir, pkgdir, distdirfiles, pkgfiles, logger)

    def regenPackages(self, chrootPath, pkgDirPath):
        """Regenerate packages and clean SYNC param"""
        pathPackages = pathJoin(chrootPath, pkgDirPath, "Packages")
        # remove Packages if it recreated
        if path.exists(pathPackages):
            os.unlink(pathPackages)
        self._update_binhost_packages()
        if path.exists(pathPackages):
            re_keywords = re.compile(
                '^(KEYWORDS|SYNC):.*$\n', re.M)
            data = readFile(pathPackages)
            data_blocks = data.split('\n\n')
            modified_blocks = [
                "%s\nKEYWORDS: amd64 x86" % re_keywords.sub('', x)
                for x in data_blocks[1:] if x.strip()]
            with writeFile(pathPackages) as f:
                f.write("\n\n".join(data_blocks[:1] + modified_blocks))

    def binaryCleaning(self):
        """Clean binary repository"""
        # imported from calculate_assemble
        chrootPath = self.clVars.Get('cl_builder_path')
        pkgDir = pathJoin(chrootPath,
                          self.clVars.Get('cl_builder_pkgdir'))
        dbPkg = pathJoin(chrootPath, 'var/db/pkg')

        logfile = self._get_log_file()
        logger = log("binary_cleanpkg.log", filename=logfile,
                     formatter="%(asctime)s - %(message)s")

        try:
            if not path.exists(dbPkg):
                os.makedirs(dbPkg)
            if not path.exists(pkgDir):
                os.makedirs(pkgDir)
            if path.exists(dbPkg) and path.exists(pkgDir):
                # get pkg list from distro
                pkgList = \
                    reduce(lambda x, y: x + y,
                           map(lambda x: map(
                               lambda z: path.join(x, "%s.tbz2" % z),
                               os.listdir(path.join(dbPkg, x))),
                               os.listdir(dbPkg)), [])
                # get binary packages
                binList = \
                    reduce(lambda x, y: x + y,
                           map(lambda x: map(
                               lambda z: path.join(x, z)[len(pkgDir) + 1:],
                               os.listdir(path.join(x))),
                               filter(lambda x: path.isdir(x),
                                      map(lambda x: path.join(pkgDir, x),
                                          os.listdir(pkgDir)))), [])

                # remove files which in binary and not in db/pkg
                removeList = list(set(binList) - set(pkgList))
                if removeList:
                    removelist_str = ",".join(
                        path.basename(x) for x in removeList)
                    logger.info(removelist_str)
                map(lambda x: os.unlink(x),
                    map(lambda x: pathJoin(pkgDir, x),
                        removeList))

                # remove empty directories
                map(lambda x: os.rmdir(x),
                    filter(lambda x: path.isdir(x) and not os.listdir(x),
                           map(lambda x: path.join(pkgDir, x),
                               os.listdir(pkgDir))))

                self.regenPackages(chrootPath, pkgDir[len(chrootPath):])
        except OSError as e:
            raise BuilderError(str(e))
        return True

    def raiseOutdate(self):
        """
        Установить флаг данные о репозиториях устарели (необходим для выполнения
        eix-update и прочих команд обновляющих кэш
        :return:
        """
        self.clVars.Set('cl_builder_outdate_set', 'on', force=True)

    def apply_branch_variables(self):
        """
        Применить значение переменной для выбора веток репозиториев
        при обновлении
        """
        self.clVars.Set('update.cl_update_branch_name',
                        self.clVars.Get('builder.cl_builder_branch_name'))
        return True

    def isohybrid(self, image_file):
        """
        Преобразовать ISO образ в гибридный
        :param image_file: iso образ
        :return:
        """
        isohybrid = getProgPath("/usr/bin/isohybrid")
        if not isohybrid:
            raise BuilderError(_("Isohybrid utility not found"))
        if not path.exists(image_file):
            raise BuilderError(_("Image not found"))
        if self.clVars.Get('os_builder_arch_machine') == 'x86_64':
            cmd = [isohybrid, "--uefi", image_file]
        else:
            cmd = [isohybrid, image_file]
        isohybrid_process = process(*cmd)
        return isohybrid_process.success()

    def _list_file(self, iso_file):
        """
        .list файл по iso файлу
        :param iso_file:
        :return:
        """
        if iso_file.endswith(".iso"):
            return "%s.list" % iso_file[:-4]
        else:
            return "%s.list" % iso_file

    def _digest_file(self, iso_file):
        return "%s.DIGESTS" % iso_file

    def create_package_list(self, chroot, iso_file):
        """
        Создает список установленных пакетов в chroot директории и сохраняет в
        iso_file
        :return:
        """
        pkgdir = path.join(chroot, 'var/db/pkg')
        list_file = self._list_file(iso_file)
        with writeFile(list_file) as f:
            f.write("\n".join(sorted(
                x.atom for x in InstalledPackageInfo.get_install_packages(
                    pkg_dir=pkgdir))))
        return True

    def create_digest(self, isofile):
        """
        Создать контрольную сумму для файла
        """
        template = """# %(alg)s HASH\n%(digest)s  %(filename)s\n"""
        digestfile = self._digest_file(isofile)
        try:
            with writeFile(digestfile) as f:
                f.writelines(map(lambda x: template % {
                    'alg': x.upper(),
                    'digest':
                        process("%ssum" % x, isofile).read().partition(' ')[0],
                    'filename': path.basename(isofile)}, ["md5", "sha1"]))
        except (IOError, OSError):
            return False
        return True

    @property
    def chroot_process(self):
        chroot_cmd = getProgPath("/usr/bin/chroot")
        if not chroot_cmd:
            raise BuilderError(_("Chroot command not found"))
        arch = self.clVars.Get('builder.os_builder_arch_machine')
        local_arch = self.clVars.Get('os_arch_machine')
        process_func = process
        if arch != local_arch:
            process_func = partial(process_func, "/usr/bin/linux32")
        return partial(process_func, chroot_cmd)

    def get_default_emerge_opts(self):
        dv = self.clVars.Get('builder.cl_builder_linux_datavars')
        if dv:
            deo = dv.Get('cl_emerge_default_opts')
        else:
            deo = super(Builder, self).get_default_emerge_opts()
        return deo

    def emerge(self, builder_path, use, *params):
        """
        Выполнить сборку пакета
        """
        deo = self.get_default_emerge_opts()
        extra_params = [x for x in params if x.startswith("-")]
        packages = [x for x in params if not x.startswith("-")]
        with EmergeParser(self.chrootize(builder_path, EmergeCommand(
                packages, emerge_default_opts=deo,
                extra_params=extra_params, use=use))) as emerge:
            try:
                emerge.question.action = lambda x: False
                emerge.run()
                if not emerge.install_packages.list:
                    return True
            except EmergeError:
                self.emerge_cache.drop_cache("Emerge error")
                self._display_error(emerge.prepare_error)
                raise
            self._startEmerging(emerge)
        return True

    def remove_list_digest(self, isofile):
        for fn in (self._digest_file(isofile), self._list_file(isofile)):
            if path.exists(fn):
                os.unlink(fn)
        return True

    def recount_files(self, builder_path, exclude_files):
        """
        Посчитать файлы в собираемой системе и исключить exclude файлы
        :param builder_path:
        :param exclude_files:
        :return:
        """
        builder_path = builder_path.encode('utf-8')
        all_count = countFiles(builder_path)
        exclude_sum = sum(countFiles(path.join(builder_path, x))
                          for x in exclude_files)
        self.clVars.Set('os_builder_linux_files', str(all_count - exclude_sum),
                        force=True)
        return True

    def create_dev_nodes(self, directory):
        """Create nodes for dev http://dev.gentoo.org/~a3li/openrc.txt"""
        devPath = path.join(directory, "dev")
        # clean dev
        for pathname, dirs, files in os.walk(devPath, topdown=False):
            map(lambda x: os.unlink(path.join(pathname, x)), files)
            map(lambda x: os.unlink(x) if path.islink(x) else os.rmdir(x),
                map(lambda x: path.join(pathname, x), dirs))
        for node, mode, dmode, major, minor in [
            ("console", 0600, stat.S_IFCHR, 5, 1),
            ("tty1", 0600, stat.S_IFCHR, 4, 1),
            ("null", 0666, stat.S_IFCHR, 1, 3),
            ("zero", 0666, stat.S_IFCHR, 1, 5)]:
            nodePath = path.join(devPath, node)
            os.mknod(nodePath, mode | dmode, os.makedev(major, minor))
            os.chmod(nodePath, mode)
        return True

    def check_build_run(self):
        """
        Проверить повторный запуск
        """
        build_id = self.clVars.Get('cl_builder_id')
        names = self.Method.All
        pid = os.getpid()
        filter_func = lambda x: ('id' in x and x['id'] == build_id
                                 and x['name'] in names
                                 and x['os_pid'] != pid)
        dv = self.clVars
        if any(search_worked_process2(dv, filter_func=filter_func)):
            raise BuilderError(_("Builder action for %s is already running. "
                                 "Try to run later.") % build_id)
        return True

    def check_chroot_run(self):
        """
        Проверить наличие chroot процессов
        :return:
        """
        builder_path = self.clVars.Get('cl_builder_path')
        programs = getRunCommands(chroot=builder_path)
        if programs:
            raise BuilderError(
                _("Chrooted {cmd} has already run into {id}").format(
                    cmd=programs[0].split('\x00')[0],
                    id=self.clVars.Get('cl_builder_id')
                ))
        return True

    def update_menu(self, dn):
        with IsoDistributive(dn) as iso:
            d = iso.getDirectory()
            root_path = path.relpath(dn, d)
            self.clVars.Set('cl_builder_iso_path', dn, force=True)
            self.clVars.Set('cl_builder_target', iso, force=True)
            self.clVars.Set('cl_builder_path', d, force=True)
            self.clVars.getInfo('cl_builder_videodrv_set').autodetect = True
            self.clVars.Invalidate('cl_builder_videodrv_set', force=True)
            from calculate.lib.cl_template import templateFunction

            templateFunction.installProg = {}
            templateFunction.installCategory = []
            self.applyTemplates(d, False, False, root_path)
        return True

    def remove_flash_tmp(self):
        try:
            image = self.clVars.Get('cl_builder_image')
            if image:
                image.close()
        except DistributiveError:
            pass
        flashTmp = path.join(self.clVars.Get('cl_builder_flash_path'), "tmp")
        if path.exists(flashTmp) and not listDirectory(flashTmp):
            try:
                os.rmdir(flashTmp)
            except (OSError, IOError) as e:
                self.printWARNING(str(e))
        return True

    def remount_rw(self, dn):
        """
        Перемонтировать каталог для чтения/записи (используется для flash)
        :param dn: каталог
        :return:
        """
        if not dn:
            raise BuilderError(_("Failed to remount Flash drive"))

        p = process('/bin/mount', '-o', 'remount,rw', dn, stderr=STDOUT)
        if p.failed():
            self.printERROR(p.read().strip())
            return False
        if not check_rw(dn):
            raise BuilderError("Selected device is read-only")
        return True

    def sync_vmlinuz(self, flash_dn):
        """
        Извлечение ядер из iso образов
        :return:
        """
        boot_dn = path.join(flash_dn, "boot")
        for fn in listDirectory(boot_dn, fullPath=True):
            if fn not in ("vmlinuz", "initrd"):
                try:
                    if not path.isdir(fn):
                        os.unlink(fn)
                except (IOError, OSError) as e:
                    self.printERROR(str(e))
                    raise BuilderError(
                        _("Failed to clean /boot directory on Flash drive"))
        for data in self.clVars.Get('cl_builder_image_data'):
            isofn = data[2]
            vmlinuz_orig = data[3]
            vmlinuz = data[4]
            initrd_orig = data[5]
            initrd = data[6]
            with IsoDistributive(isofn) as iso:
                dn = iso.getDirectory()
                vmlinuz_orig = path.join(dn, "boot", vmlinuz_orig)
                initrd_orig = path.join(dn, "boot", initrd_orig)
                vmlinuz = path.join(boot_dn, vmlinuz)
                initrd = path.join(boot_dn, initrd)
                try:
                    open(vmlinuz, 'w').write(open(vmlinuz_orig, 'rb').read())
                    open(initrd, 'w').write(open(initrd_orig, 'rb').read())
                except (IOError, OSError) as e:
                    self.printERROR(str(e))
                    raise BuilderError(_("Failed to extract kernel from %s")
                                       % isofn)
        return True

    def iso_migrate(self, flash_path):
        """
        Миграция образов из flash:/iso в flash:/linux
        :param flash_path:
        :return:
        """
        try:
            old_path = path.join(flash_path, "iso")
            new_path = path.join(flash_path, "linux")
            if path.exists(old_path):
                if listDirectory(old_path):
                    if path.exists(new_path):
                        for fn in listDirectory(old_path):
                            old_fn = path.join(old_path, fn)
                            new_fn = path.join(new_path, fn)
                            if path.exists(new_fn):
                                os.unlink(new_fn)
                            os.rename(old_fn, new_fn)
                    else:
                        os.rename(old_path, new_path)
                else:
                    os.rmdir(old_path)
        except OSError as e:
            self.printWARNING(
                _("Failed to move the ISO images directory on the Flash"))
            self.printWARNING(str(e))
        return True

    def _get_default_params(self, drvs):
        """
        Получить параметры загрузки для системы LiveHDD
        """
        proprietary = ["nvidia", "fglrx"]
        video = self.clVars.Get('cl_builder_x11_video_drv')
        locale = self.clVars.Get('cl_builder_locale_lang')
        timezone = self.clVars.Get('cl_builder_timezone')
        params = ["%s:%s" % (CmdlineParams.Locale, locale),
                  "%s:%s" % (CmdlineParams.Timezone, timezone)]
        resolution = self.clVars.Get('cl_builder_x11_resolution')
        if resolution and resolution != "auto":
            params.append("%s:%s" % (CmdlineParams.Resolution, resolution))
        if video != "default" and (
                video not in proprietary or Variable.isTrue(drvs)):
            params.append("%s:%s" % (CmdlineParams.Video, video))
            current_video = self.clVars.Get('os_x11_video_drv')
            if current_video == video and video != "default":
                composite = self.clVars.Get('cl_builder_x11_composite')
                params.append("%s:%s" % (CmdlineParams.Composite, composite))
        calculate_param = ",".join(params)
        if self.clVars.GetBool('cl_builder_docache_set'):
            return "%s docache" % calculate_param
        else:
            return calculate_param

    def create_iso_grub_cfg(self, dn):
        gc = GrubCommand()
        content = []
        for label, iso, splash, drvs in self.clVars.ZipVars(
                'cl_builder_image_label',
                'cl_builder_image_iso',
                'cl_builder_image_splash',
                'cl_builder_image_drivers'):
            default_params = self._get_default_params(drvs)
            entry = (
                "menuentry '%(label)s' {\n"
                "\tset isofile=%(iso)s\n"
                "\tloopback loop $isofile\n"
                "\tlinux (loop)/boot/vmlinuz root=live "
                "iso-scan/filename=$isofile quiet %(splash)s "
                "calculate=%(defs)s\n"
                "\tinitrd (loop)/boot/initrd\n}\n\n" % {
                    'label': label,
                    'iso': gc.get_relpath(iso),
                    'splash': templateFunction.splash_cmd(splash),
                    'defs': default_params
                })
            content.append(entry)
        fn_grubcfg = path.join(dn, 'grub.cfg')
        try:
            write_content = ("\n".join(content)).strip() + "\n"
            if write_content.strip():
                with writeFile(fn_grubcfg) as f:
                    f.write(write_content)
            else:
                self.clear_iso_grub_cfg(dn)
        except IOError:
            raise BuilderError(_("Failed to write %s") % fn_grubcfg)
        return True

    def clear_iso_grub_cfg(self, dn):
        if dn:
            cfg_fn = path.join(dn, 'grub.cfg')
            if path.exists(cfg_fn):
                try:
                    os.unlink(cfg_fn)
                except OSError:
                    raise BuilderError(_("Failed to remove %s") % cfg_fn)
        return True

    def setup_package(self, package):
        """
        Обновить конфигурационные файлы системы
        """
        clVars = DataVars()
        try:
            clVars.importData()
            clVars.flIniFile()
            clVars.Set("cl_root_path", "/", True)

            clVars.Set("cl_merge_pkg", [package], True)
            clVars.Set("cl_action", 'merge', True)
            for copyvar in ("cl_dispatch_conf", "cl_verbose_set",
                            "cl_template_path_use",
                            "builder.cl_livemenu_path",
                            "builder.cl_builder_livemenu_path"):
                clVars.Set(copyvar, self.clVars.Get(copyvar), force=True)
            useClt = self.clVars.Get('cl_template_clt_set') in (True, "on")

            # используем объект шаблонов
            # с clt шаблонами, clt фильтром, без использования postDispatchConf
            clTempl = ChainProgressTemplate(
                self.startTask,
                self.endTask,
                self.setProgress,
                clVars, cltObj=useClt,
                cltFilter=True,
                printSUCCESS=self.printSUCCESS,
                printERROR=self.printERROR,
                printWARNING=self.printWARNING,
                askConfirm=self.askConfirm,
                dispatchConf=self.dispatchConf,
                printWarning=False)
            clTempl.applyTemplates()
        finally:
            clVars.close()
            self.endTask()
        return True