Множественные изменения

Исправлен дисковый спул
Добавлено вычисление files_num
.DIGEST и .list файл
Prelink перенесён в image
Добавлена поддержка сборки 32 из 64
Удаление дерева портежей перед запаковкой
isohybrid
режим сжатия
обновление до fallback профилей
изменено обновление до веток и тэгов

pym/builder/builder.py

@@ -23,18 +23,19 @@ from calculate.lib.utils.files import (
     pathJoin, PercentProgress, getProgPath, process, STDOUT, removeDir,
     makeDirectory, writeFile, readLinesFile, chmod, chown, FilePermission,
     find, FindFileType, removeFileWithEmptyDirectory,
-    copyWithPath)
+    copyWithPath, countFiles)
 from calculate.lib.utils.git import Git
 from calculate.lib.utils.portage import Layman, EmergeLog, EmergeLogNamedTask, \
     InstalledPackageInfo, EbuildInfoError, EbuildInfo, ChrootEix, \
     get_packages_files_directory, get_manifest_files_directory
 from calculate.update.emerge_parser import EmergeParser, \
-    EmergeError, ChrootEmergeCommand, ChrootCommandExecutor, RevdepPercentBlock, \
-    EmergeCache
+    EmergeError, EmergeCommand, Chroot, Linux32, CommandExecutor, \
+    RevdepPercentBlock, EmergeCache
 from calculate.lib.cl_log import log
 from .build_storage import Build
 from calculate.update.update import Update
 from calculate.install.distr import Distributive
+from functools import partial
 import os
 from os import path
 from .datavars import BuilderError
@@ -44,7 +45,6 @@ from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, _)
 setLocalTranslate('cl_builder3', sys.modules[__name__])
 __ = getLazyLocalTranslate(_)
 
-
 class Builder(Update):
     """Основной объект для выполнения действий связанных со сборкой системы
 
@@ -57,7 +57,7 @@ class Builder(Update):
 
     def mount_target(self, target):
         dir_distro = target.convertToDirectory()
-        dir_distro.mountSystemDirectories()
+        dir_distro.mountSystemDirectories(skip=())
         return True
 
     def umount_system(self, target):
@@ -115,20 +115,6 @@ class Builder(Update):
         image.eventPrepareIso.connect(self.prepare_iso)
         return True
 
-    #def test_builder(self):
-    #    path_chroot = self.clVars.Get('cl_builder_path')
-    #    d1 = dict(zip(
-    #        map(lambda x: pathJoin(path_chroot, x),
-    #            self.clVars.Get('cl_builder_repository_location')),
-    #            self.clVars.Get('cl_builder_repository_name')))
-    #    print d1
-    #    d2 = dict(zip(self.clVars.Get('update.cl_update_rep_path'),
-    #                  self.clVars.Get('update.cl_update_rep_name')))
-    #    print d2
-    #    print "!!!!!!",self.clVars.Get('cl_builder_linux_datavars').Get('cl_update_rep_path')
-    #    print "!!!!!!",self.clVars.Get('cl_builder_profile_datavars').Get('cl_update_rep_path')
-    #    return True
-
     def remove_repositories(self, repname):
         dv = self.clVars.Get('cl_builder_linux_datavars')
         rpath = dv.Select("cl_update_rep_path",
@@ -225,10 +211,9 @@ class Builder(Update):
                                        printERROR=self.printERROR)
 
             def execute_command(cmd, lang):
-                chroot_cmd = getProgPath("/usr/bin/chroot", )
                 chroot_path = self.clVars.Get('cl_builder_path')
-                return process(chroot_cmd, chroot_path,
-                               cmd, lang=lang, envdict=dict(os.environ))
+                return self.chroot_process(
+                    chroot_path, cmd, lang=lang, envdict=dict(os.environ))
 
             # замена выполения команд: вместо оычного запуска - запуск через
             # /usr/bin/chroot
@@ -264,9 +249,9 @@ class Builder(Update):
 
     def _regenCache_process(self, progname, repname, cpu_num):
         chroot_path = self.clVars.Get('builder.cl_builder_path')
-        return process("/usr/bin/chroot", chroot_path,
-                       progname, "--repo=%s" % repname, "--update",
-                       "--jobs=%s" % cpu_num, stderr=STDOUT)
+        return self.chroot_process(chroot_path,
+                                   progname, "--repo=%s" % repname, "--update",
+                                   "--jobs=%s" % cpu_num, stderr=STDOUT)
 
     def clear_log(self, builder_id_path):
         logname = "build-%s" % builder_id_path
@@ -283,6 +268,20 @@ class Builder(Update):
         mainlog = self.clVars.Get('core.cl_log_path')
         return path.join(mainlog, logname)
 
+    def chrootize(self, chroot_path, cmd):
+        """
+        :param chroot_path:
+        :param cmd:
+        :return:
+        """
+        arch = self.clVars.Get('os_builder_arch_machine')
+        local_arch = self.clVars.Get('os_arch_machine')
+        # упрощенная проверка так как только 64 может собирать 32
+        if arch != local_arch:
+            return Linux32(Chroot(chroot_path, cmd))
+        else:
+            return Chroot(chroot_path, cmd)
+
     def emerge_ask(self, pretend, *params):
         """
         Вывести информацию об обновлении
@@ -292,11 +291,11 @@ class Builder(Update):
         packages = [x for x in params if not x.startswith("-")]
         chroot_path = self.clVars.Get('cl_builder_path')
         logfile = self._get_log_file()
-        with EmergeParser(ChrootEmergeCommand(chroot_path,
-                                              list(packages),
-                                              emerge_default_opts=deo,
-                                              extra_params=param,
-                                              logfile=logfile)) as emerge:
+        with EmergeParser(self.chrootize(chroot_path, EmergeCommand(
+                list(packages),
+                emerge_default_opts=deo,
+                extra_params=param,
+                logfile=logfile))) as emerge:
             try:
                 emerge.question.action = lambda x: False
                 emerge.run()
@@ -330,10 +329,10 @@ class Builder(Update):
         deo = self.clVars.Get('cl_emerge_default_opts')
         chroot_path = self.clVars.Get('cl_builder_path')
         logfile = self._get_log_file()
-        with EmergeParser(
-                ChrootEmergeCommand(chroot_path, ["--depclean", "--ask=y"],
-                                    logfile=logfile,
-                                    emerge_default_opts=deo)) as emerge:
+        with EmergeParser(self.chrootize(chroot_path, EmergeCommand(
+                ["--depclean", "--ask=y"],
+                logfile=logfile,
+                emerge_default_opts=deo))) as emerge:
             try:
                 emerge.question.action = lambda x: False
                 emerge.run()
@@ -363,10 +362,7 @@ class Builder(Update):
 
         if not cmdpath:
             return "skip"
-            #raise BuilderError(_("The %s tool is not found")%command)
-        p = process("/usr/bin/chroot",
-                    *([builder_path, cmdpath] + list(params)),
-                    stderr=STDOUT)
+        p = self.chroot_process(builder_path, cmdpath, *params, stderr=STDOUT)
         with writeFile(self._get_log_file()) as f:
             f.write(p.read())
         if p.failed():
@@ -459,12 +455,12 @@ class Builder(Update):
         :return: список пакетов
         """
         if atom not in self.pretend_package_list:
-            emerge = EmergeParser(ChrootEmergeCommand(
-                chroot_path, [atom], extra_params=['-pv', '--ask=n'],
-                logfile=logfile))
-            emerge.question.default_answer = "n"
-            emerge.run()
-            self.pretend_package_list[atom] = list(emerge.install_packages.list)
+            with EmergeParser(self.chrootize(chroot_path, EmergeCommand(
+                    [atom], extra_params=['-pv', '--ask=n'],
+                    logfile=logfile))) as emerge:
+                emerge.question.default_answer = "n"
+                emerge.run()
+                self.pretend_package_list[atom] = list(emerge.install_packages.list)
         return self.pretend_package_list[atom]
 
     def _display_video_install_package(self, package_list, drv_name):
@@ -539,9 +535,9 @@ class Builder(Update):
             self.startTask(_("Fetching binary packages and sources tarballs") %
                            [x for x in package_list if x['PN'] == drv_name][0])
 
-            ef = EmergeFetcher(ChrootEmergeCommand(
-                builder_path, ["=%s" % x for x in package_list],
-                extra_params=["-Of", "--ask=n"], logfile="%s.2" % logfile))
+            ef = EmergeFetcher(self.chrootize(builder_path, EmergeCommand(
+                ["=%s" % x for x in package_list],
+                extra_params=["-Of", "--ask=n"], logfile="%s.2" % logfile)))
             try:
                 for package in ef:
                     pkg_name = str(package)
@@ -656,10 +652,8 @@ class Builder(Update):
         :return:
         """
         eselect_command = "/usr/bin/eselect"
-        p = process("/usr/bin/chroot",
-                    *[builder_path, eselect_command, "--colour=yes",
-                      "news", "read", "new"],
-                    stderr=STDOUT)
+        p = self.chroot_process(builder_path, eselect_command, "--colour=yes",
+                                "news", "read", "new", stderr=STDOUT)
         self.printPre(convert_console_to_xml(
             p.read()).replace("  ", "  "))
         return True
@@ -689,10 +683,9 @@ class Builder(Update):
         logfile = self._get_log_file()
         if not cmd_path:
             raise BuilderError(_("Failed to find the %s command") % cmd)
-        with EmergeParser(
-                ChrootCommandExecutor(builder_path,
-                                      cmd_path, params,
-                                      logfile=logfile)) as emerge:
+        with EmergeParser(self.chrootize(builder_path, CommandExecutor(
+                cmd_path, params,
+                logfile=logfile))) as emerge:
             self._startEmerging(emerge)
         return True
 
@@ -704,8 +697,8 @@ class Builder(Update):
         logfile = self._get_log_file()
         if not cmd_path:
             raise BuilderError(_("Failed to find the %s command") % cmd)
-        with EmergeParser(ChrootCommandExecutor(
-                builder_path, cmd_path, params, logfile=logfile)) as emerge:
+        with EmergeParser(self.chrootize(builder_path, CommandExecutor(
+                builder_path, cmd_path, params, logfile=logfile))) as emerge:
             revdep = RevdepPercentBlock(emerge)
             self.addProgress()
             revdep.add_observer(self.setProgress)
@@ -738,9 +731,8 @@ class Builder(Update):
                 for perc in p.progress():
                     self.setProgress(perc)
             else:
-                p = process(
-                    "/usr/bin/chroot", chroot_path,
-                    layman, "-s", repname, stderr=STDOUT)
+                p = self.chroot_process(
+                    chroot_path, layman, "-s", repname, stderr=STDOUT)
             if p.failed():
                 raise BuilderError(
                     _("Failed to update the {rname} repository").format(
@@ -794,5 +786,122 @@ class Builder(Update):
         при обновлении
         """
         self.clVars.Set('update.cl_update_branch_name',
-            self.clVars.Get('builder.cl_builder_branch_name'))
+                        self.clVars.Get('builder.cl_builder_branch_name'))
+        return True
+
+    def isohybrid(self, image_file):
+        """
+        Преобразовать ISO образ в гибридный
+        :param image_file: iso образ
+        :return:
+        """
+        isohybrid = getProgPath("/usr/bin/isohybrid")
+        if not isohybrid:
+            raise BuilderError(_("Isohybrid utility not found"))
+        if not path.exists(image_file):
+            raise BuilderError(_("Image not found"))
+        if self.clVars.Get('os_builder_arch_machine') == 'x86_64':
+            cmd = [isohybrid, "--uefi", image_file]
+        else:
+            cmd = [isohybrid, image_file]
+        isohybrid_process = process(*cmd)
+        return isohybrid_process.success()
+
+    def _list_file(self, iso_file):
+        """
+        .list файл по iso файлу
+        :param iso_file:
+        :return:
+        """
+        if iso_file.endswith(".iso"):
+            return "%s.list" % iso_file[:-4]
+        else:
+            return "%s.list" % iso_file
+
+    def _digest_file(self, iso_file):
+        return "%s.DIGEST" % iso_file
+
+    def create_package_list(self, chroot, iso_file):
+        """
+        Создает список установленных пакетов в chroot директории и сохраняет в
+        iso_file
+        :return:
+        """
+        pkgdir = path.join(chroot, 'var/db/pkg')
+        list_file = self._list_file(iso_file)
+        with writeFile(list_file) as f:
+            f.write("\n".join(sorted(
+                x.atom for x in InstalledPackageInfo.get_install_packages(
+                    pkg_dir=pkgdir))))
+        return True
+
+    def create_digest(self, isofile):
+        """
+        Создать контрольную сумму для файла
+        """
+        template = """# %(alg)s HASH\n%(digest)s  %(filename)s\n"""
+        digestfile = self._digest_file(isofile)
+        try:
+            with writeFile(digestfile) as f:
+                f.writelines(map(lambda x: template % {
+                    'alg': x.upper(),
+                    'digest':
+                        process("%ssum" % x, isofile).read().partition(' ')[0],
+                    'filename': path.basename(isofile)}, ["md5", "sha1"]))
+        except (IOError, OSError):
+            return False
+        return True
+
+    @property
+    def chroot_process(self):
+        chroot_cmd = getProgPath("/usr/bin/chroot")
+        if not chroot_cmd:
+            raise BuilderError(_("Chroot command not found"))
+        arch = self.clVars.Get('builder.os_builder_arch_machine')
+        local_arch = self.clVars.Get('os_arch_machine')
+        process_func = process
+        if arch != local_arch:
+            process_func = partial(process_func, "/usr/bin/linux32")
+        return partial(process_func, chroot_cmd)
+
+    def emerge(self, builder_path, use, *params):
+        """
+        Выполнить сборку пакета
+        """
+        deo = self.clVars.Get('cl_emerge_default_opts')
+        extra_params = [x for x in params if x.startswith("-")]
+        packages = [x for x in params if not x.startswith("-")]
+        with EmergeParser(self.chrootize(builder_path, EmergeCommand(
+                packages, emerge_default_opts=deo,
+                extra_params=extra_params, use=use))) as emerge:
+            try:
+                emerge.question.action = lambda x: False
+                emerge.run()
+                if not emerge.install_packages.list:
+                    return True
+            except EmergeError:
+                self.emerge_cache.drop_cache("Emerge error")
+                self._display_error(emerge.prepare_error)
+                raise
+            self._startEmerging(emerge)
+        return True
+
+    def remove_list_digest(self, isofile):
+        for fn in (self._digest_file(isofile), self._list_file(isofile)):
+            if path.exists(fn):
+                os.unlink(fn)
+        return True
+
+    def recount_files(self, builder_path, exclude_files):
+        """
+        Посчитать файлы в собираемой системе и исключить exclude файлы
+        :param builder_path:
+        :param exclude_files:
+        :return:
+        """
+        all_count = countFiles(builder_path)
+        exclude_sum = sum(countFiles(path.join(builder_path, x))
+                          for x in exclude_files)
+        self.clVars.Set('os_builder_linux_files', str(all_count - exclude_sum),
+                        force=True)
         return True

pym/builder/drive_spool.py

@@ -46,7 +46,9 @@ class DriveSpool(object):
 
     @staticmethod
     def check_device(device):
-        return path.exists(device)
+        if device.startswith("/dev"):
+            return path.exists(device)
+        return True
 
     def lock_device(self, device):
         if self.check_device(device) and not isMount(device):

pym/builder/utils/cl_builder_image.py

@@ -49,6 +49,12 @@ class ClBuilderImageAction(Action):
          'message': __("Configuring build"),
          'method': 'Builder.applyTemplates(cl_builder_target,False,False,None)',
          },
+        {'name': 'prelink',
+         'message': __("Executing prelink"),
+         'method': 'Builder.chroot_command(cl_builder_path,"prelink",'
+                   '"-afmR")',
+         'condition': lambda Get: Get('cl_builder_prelink_set') == 'on'
+         },
         # получить видеодрайверы
         {'name': 'fetch_video_drivers',
          'group': __("Fetching video drivers"),
@@ -91,6 +97,10 @@ class ClBuilderImageAction(Action):
              {'name': 'squash_action',
               'method': 'Builder.set_builder_action("squash")',
               },
+             {'name': 'count_files',
+              'method': 'Builder.recount_files(cl_builder_path,'
+                        'cl_builder_squash_exclude)'
+              },
              {'name': 'apply_template',
               'message': __("Configuring squash"),
               # наложить шаблоны в развернутый дистрибутив
@@ -100,10 +110,28 @@ class ClBuilderImageAction(Action):
              {'name': 'squash_action',
               'method': 'Builder.set_builder_action("iso")',
               },
-             # распаковка дистрибутива
+             # сборка iso
+             {'name':'remove_list_digest',
+              'method': "Builder.remove_list_digest(cl_builder_image_filename)"
+              },
              {'name': 'unpack',
               'message': __("Pack squash"),
-              'method': 'Install.unpack(cl_builder_target, cl_builder_image, "0")',
+              'method': 'Install.unpack(cl_builder_target, cl_builder_image,'
+                        '"0")',
+              },
+             {'name': 'isohybrid',
+              'message': __("Appling isohybrid feature for image"),
+              'method': 'Builder.isohybrid(cl_builder_image_filename)',
+              'condition': lambda Get: Get('cl_builder_isohybrid_set') == 'on'
+              },
+             {'name': 'create_package_list',
+              'message': __("Creating package list"),
+              'method': 'Builder.create_package_list(cl_builder_path,'
+                        'cl_builder_image_filename)',
+              },
+             {'name': 'create_digest',
+              'message': __("Creating DIGEST for image"),
+              'method': 'Builder.create_digest(cl_builder_image_filename)',
               },
              {'name': 'umount_system!:mount_system',
               'warning': _("Restore build system resources"),

pym/builder/utils/cl_builder_profile.py

@@ -55,6 +55,9 @@ class ClBuilderProfileAction(Action):
          'method': 'Builder.invalidateVariables("cl_builder_profile_storage")',
          'depend': Tasks.has('migrate_repository')
          },
+        {'name': 'drop_binhosts',
+         'method': 'Builder.drop_binhosts(cl_builder_profile_datavars)'
+         },
         {'name': 'reconfigure_vars',
          'method': 'Builder.reconfigureProfileVars(cl_builder_profile_datavars,'
                    'cl_builder_path)'

pym/builder/utils/cl_builder_update.py

@@ -81,11 +81,41 @@ class ClBuilderUpdateAction(Action):
         {'name': 'reps_synchronization',
          'group': __("Repositories synchronization"),
          'tasks': [
+             # запасная синхронизация, в ходе которой ветки обновляются до
+             # master
+             {'name': 'sync_reps_fallback',
+              'foreach': 'cl_builder_sync_rep',
+              'message':
+                  __("Fallback syncing the {eachvar:capitalize} repository"),
+              'method': 'Builder.syncRepositories(eachvar)',
+              'condition': lambda Get: (
+                  Get('cl_builder_binary_set') == "on" and
+                  not Get('update.cl_update_binhost_data')[0])
+             },
+             # обновление переменных информации из binhost
+             {'name': 'invalidate_vars',
+              'method':
+                  'Builder.invalidateVariables("cl_builder_linux_datavars")',
+              'condition': lambda Get: (
+                  Get('cl_builder_binary_set') == "on" and
+                  not Get('update.cl_update_binhost_data')[0])
+              },
+             {'name': 'binhost_unavailable',
+              'error': __("Binhost is unavailable"),
+              'condition': lambda Get:(
+                  Get('cl_builder_binary_set') == "on" and
+                  not Get('update.cl_update_binhost_data')[0])
+              },
              {'name': 'sync_reps',
               'foreach': 'cl_builder_sync_rep',
               'message': __("Syncing the {eachvar:capitalize} repository"),
               'method': 'Builder.syncRepositories(eachvar)',
+              'condition': lambda Get: Get('cl_builder_sync_rep')
              },
+             {'name': 'check_binhost',
+              'method': 'Builder.check_binhost(False)',
+              'condition': lambda Get: Get('cl_builder_binary_set') == "on"
+              },
              {'name': 'sync_other_reps',
               'foreach': 'builder.cl_builder_other_rep_name',
               'message': __("Syncing the {eachvar:capitalize} repository"),
@@ -169,7 +199,9 @@ class ClBuilderUpdateAction(Action):
              {'name': 'changed_packages',
               'message': __("Calculating dependencies"),
               'method': 'Builder.rebuild_changed_packages(cl_builder_path,'
-                        'cl_builder_repository_data)'
+                        'cl_builder_repository_data)',
+              'condition': lambda Get:
+                  Get('cl_builder_rebuild_changed_set') == 'on'
               }
          ]
         },
@@ -217,7 +249,7 @@ class ClBuilderUpdateAction(Action):
          'tasks': [
              {'name': 'update_world:module_rebuild',
               'message': __('Updating Kernel modules'),
-              'method': 'Builder.emerge("","@module-rebuild")',
+              'method': 'Builder.emerge(cl_builder_path,"","@module-rebuild")',
               'condition': was_installed('sys-kernel/.*source',
                                          EmergeMark.KernelModules),
               'decoration': 'Builder.update_task("%s")' %
@@ -225,14 +257,16 @@ class ClBuilderUpdateAction(Action):
               },
              {'name': 'update_world:x11_module_rebuild',
               'message': __('Updating X.Org server modules'),
-              'method': 'Builder.emerge("","@x11-module-rebuild")',
+              'method': 'Builder.emerge(cl_builder_path,"",'
+                        '"@x11-module-rebuild")',
               'condition': was_installed('x11-base/xorg-server',
                                          EmergeMark.XorgModules),
               'decoration': 'Builder.update_task("%s")' % EmergeMark.XorgModules
               },
              {'name': 'update_world:preserved_rebuild',
               'message': __('Updating preserved libraries'),
-              'method': 'Builder.emerge("","@preserved-rebuild")',
+              'method': 'Builder.emerge(cl_builder_path,"",'
+                        '"@preserved-rebuild")',
               'condition': was_installed('.*', EmergeMark.PreservedLibs),
               'decoration': 'Builder.update_task("%s")' %
                             EmergeMark.PreservedLibs
@@ -256,13 +290,6 @@ class ClBuilderUpdateAction(Action):
         ],
          'depend': Tasks.has("update_other")
         },
-        {'name': 'update_other:prelink',
-         'message': __("Executing prelink"),
-         'method': 'Builder.chroot_command(cl_builder_path,"prelink",'
-                   '"-afmR")',
-         'condition': was_installed('.*', EmergeMark.Prelink),
-         'decoration': 'Builder.update_task("%s")' % EmergeMark.Prelink
-        },
         {'name': 'update_other:reading_news',
          'method': 'Builder.reading_news(cl_builder_path)'
         },

pym/builder/variables/builder.py

@@ -18,9 +18,8 @@ import sys
 from os import path
 import os
 import re
-from calculate.lib.configparser import ConfigParser
-from calculate.lib.utils.git import Git
-from calculate.lib.utils.git import GitError
+from calculate.install.variables.kernel import KernelConfig
+from calculate.lib.utils.portage import getSquashList
 from .action import Actions
 from calculate.install import distr
 from calculate.lib.utils.device import getUdevDeviceInfo, humanreadableSize
@@ -118,15 +117,12 @@ class VariableClBuilderVideodrvSet(Variable):
     """
     type = "bool"
     opt = ["--video", "-V"]
+    value = "off"
 
     def init(self):
         self.label = _("Fetch video drivers")
         self.help = _("fetch video drivers")
 
-    def get(self):
-        fn = self.Get("cl_builder_video_driver_path")
-        return "on" if path.exists(fn) else "off"
-
 class VariableClBuilderSourceFilename(Variable):
     """
     Названия файла исходного дистрибутива
@@ -136,6 +132,7 @@ class VariableClBuilderSourceFilename(Variable):
     opt = ["--source"]
     value = ""
     metavalue = "SOURCE"
+    untrusted = True
 
     def init(self):
         self.label = _("Source distribution")
@@ -148,8 +145,13 @@ class VariableClBuilderSourceFilename(Variable):
         shortname = self.Get('os_builder_linux_shortname')
         build = self.Get('os_builder_linux_build')
         arch = self.Get('os_builder_arch_machine')
+        local_arch = self.Get('os_arch_machine')
         if not build or not shortname or not arch:
             raise VariableError(_("Wrong image file"))
+        if local_arch == "i686" and arch != local_arch:
+            raise VariableError(
+                _("Unable to assemble the system {what} from {local}").format(
+                    what=arch, local=local_arch))
 
     def humanReadable(self):
         fullname = self.Get('os_builder_linux_name')
@@ -207,6 +209,9 @@ class VariableClBuilderDiskDev(Variable):
         return ds.get() or ""
 
     def check(self, value):
+        if not value:
+            raise VariableError(
+                _("You need to select a destination for build"))
         if value.startswith("/dev"):
             cnDisk = getUdevDeviceInfo(name=value).get('DEVNAME', value)
             if cnDisk not in self.Get('install.os_disk_dev'):
@@ -404,7 +409,7 @@ class VariableClBuilderNewId(BaseBuildId):
         return self.Get('cl_builder_profile_name')
 
     def check(self, value):
-        if not value:
+        if not value and self.Get('cl_builder_source_filename'):
             raise VariableError(_("Please specify the id"))
         if value in self.Get('cl_builder_storage'):
             raise VariableError(_("Assemble %s already exists")%value)
@@ -595,11 +600,16 @@ class VariableClBuilderImageFilename(Variable):
     opt = ["--iso"]
     value = ""
     metavalue = "IMAGE"
+    untrusted = True
 
     def init(self):
         self.label = _("Image name")
         self.help = _("set image name")
 
+    def check(self,value):
+        if not value:
+            raise VariableError(_("You must specify image filename"))
+
     def get(self):
         base_dn = self.Get('cl_builder_base_path')
         build_id = self.Get('cl_builder_id')
@@ -660,8 +670,11 @@ class VariableClBuilderImage(ReadonlyVariable):
     def get(self):
         image_name = self.Get('cl_builder_image_filename')
         bdn = self.Get('cl_builder_iso_path')
+        exclude_list = self.Get('cl_builder_squash_exclude')
         iso = distr.IsoDistributive(image_name, bdirectory=bdn,
-                                    vol_id=self.Get('cl_builder_iso_label'))
+                                    vol_id=self.Get('cl_builder_iso_label'),
+                                    exclude=exclude_list,
+                                    compress=self.Get('cl_builder_compress'))
         return iso
 
 
@@ -733,8 +746,28 @@ class VariableClBuilderKernelCmd(ReadonlyVariable):
     """
     value = ""
 
+class KernelInfo(ReadonlyVariable):
+    def get_current_kernel_src(self, prefix):
+        src_path = "usr/src"
+        current_linux_src = path.join(src_path, "linux")
+        symlink_kernel = path.join(prefix, current_linux_src)
+        if not path.exists(symlink_kernel) or not path.islink(symlink_kernel):
+            raise ValueError("Failed to determine current kernel version")
+        return path.join(src_path, os.readlink(symlink_kernel))
+
+class VariableClBuilderKernelConfig(KernelInfo):
+    """
+    Конфиг ядра
+    """
+    def get(self):
+        prefix = self.Get('cl_builder_path')
+        if prefix:
+            kernel_src = self.get_current_kernel_src(prefix)
+            config_path = path.join(kernel_src, ".config")
+            return KernelConfig(path.join(prefix, config_path))
+        return ""
 
-class VariableClBuilderKernelVer(ReadonlyVariable):
+class VariableClBuilderKernelVer(KernelInfo):
     """
     Текущая версия ядра
     """
@@ -781,14 +814,6 @@ class VariableClBuilderKernelVer(ReadonlyVariable):
                     return "{0}.{1}.{2}{3}".format(*match.groups())
         return ""
 
-    def get_current_kernel_src(self, prefix):
-        src_path = "usr/src"
-        current_linux_src = path.join(src_path, "linux")
-        symlink_kernel = path.join(prefix, current_linux_src)
-        if not path.exists(symlink_kernel) or not path.islink(symlink_kernel):
-            raise ValueError("Failed to determine current kernel version")
-        return path.join(src_path, os.readlink(symlink_kernel))
-
 
 class KernelData(ReadonlyVariable):
     """
@@ -883,7 +908,7 @@ class VariableClBuilderOutdateSet(ReadonlyVariable):
         return "off"
 
 
-class VariableClBuilderBranch(TableVariable):
+class VariableClBuilderBranchData(TableVariable):
     """
     Выбор веток репозиториев до которых необходимо обновиться
     """
@@ -930,28 +955,128 @@ class VariableClBuilderBranchName(Variable):
         self.label = _("Branches")
 
     def choice(self):
-        return ["master", "develop", "update"]
+        return ["master", "develop", "update", "binhost"]
 
     def get(self):
         dv = self.Get('cl_builder_linux_datavars')
+        if dv:
+            if "getbinpkg" in self.Get('cl_features'):
+                return ["binhost" for x in dv.Get('cl_update_rep_name')]
+            else:
+                branch = self.Get('cl_update_branch')
+                return [branch for x in dv.Get('cl_update_rep_name')]
+        return []
+
+class VariableClBuilderCompress(Variable):
+    """
+    Тип сжатия образа squash
+    """
+    type = "choice"
+    opt = ["-c", "--compress"]
+    metavalue = "COMPRESS"
+    untrusted = True
+
+    def init(self):
+        self.label = _("Compressor")
+        self.help = _("set the compressor")
+
+    def choice(self):
+        config_param_prefix = "CONFIG_SQUASHFS_%s"
+        kernel_config = self.Get('cl_builder_kernel_config')
+        params = {"xz": "XZ",
+                  "lzma": "XZ",
+                  "lzo": "LZO",
+                  "gzip": "ZLIB"}
+
+        def generator():
+            for compress in getSquashList():
+                if compress in params:
+                    if config_param_prefix % params[compress] in kernel_config:
+                        yield compress
+
+                else:
+                    yield compress
+
+        return list(sorted(generator()))
+
+    def get(self):
+        values = self.choice()
+        if values:
+            return values[0]
+        return ""
+
+class VariableClBuilderIsohybridSet(Variable):
+    """
+    Преобразовать полученный iso образ в гибридный
+    """
+    type = "bool"
+    opt = ["--isohybrid"]
+    value = "on"
+
+    def init(self):
+        self.help = _("create the iso image with isohybrid")
+        self.label = _("Iso hybrid feature")
+
+class VariableClBuilderKeepTreeSet(Variable):
+    """
+    Не удалять ебилды из портежей о оверлеев
+    """
+    type = "bool"
+    opt = ["--keep-tree"]
+    value = "off"
+
+    def init(self):
+        self.help = _("keep portage tree in image")
+        self.label = _("Keep portage tree")
+
+class VariableClBuilderSquashExclude(ReadonlyVariable):
+    """
+    Список файлов, которые не будут запакованы в livecd.squashfs
+    """
+    type = "list"
+
+    def get(self):
+        builder_path = self.Get('cl_builder_path')
+        keep_tree = self.GetBool('cl_builder_keep_tree_set')
+        excludes = ["", "metadata", "profiles/templates/deprecated"]
+        important = [".git", "distfiles", "packages", "eclass", "metadata",
+                     "profiles", "layout.conf"]
+
+        def generator():
+            for rep_dn in self.Get('cl_builder_repository_location'):
+                for exclude_dn in excludes:
+                    dn = pathJoin(builder_path, rep_dn, exclude_dn)
+                    for exclude_name in (x for x in listDirectory(dn)
+                                         if x not in important):
+                        yield path.join(rep_dn, exclude_dn, exclude_name)[1:]
+
+        if not keep_tree and builder_path:
+            return list(generator())
+        return []
+
+class VariableClBuilderPrelinkSet(Variable):
+    """
+    Выполнять ли prelink
+    """
+    type = "bool"
+    opt = ["--prelink"]
+
+    def init(self):
+        self.help = _("perform prelink")
+        self.label = _("Perform prelink")
+
+    def get(self):
+        return "on" if self.GetBool('cl_builder_binary_set') else "off"
+
+class VariableClBuilderRebuildChangedSet(Variable):
+    """
+    Выполнять ли prelink
+    """
+    type = "bool"
+    opt = ["--rebuild-changed-packages"]
+    value = "on"
+
+    def init(self):
+        self.help = _("rebuild changed packages")
+        self.label = _("Rebuild changed packages")
 
-        cp = ConfigParser()
-        cp.read_string(unicode(
-            dv.Get('update.cl_update_binhost_revisions')[0]))
-        repo_tag = dv.GetBool('update.cl_update_repo_tag_set')
-
-        def generateBranch():
-            git = Git()
-            for reppath, repname in zip(dv.Get('cl_update_rep_path'),
-                                        dv.Get('cl_update_rep_name')):
-                tag = cp.get("vcs", repname, fallback="master")
-                try:
-                    if (repo_tag or git.isTagRepository(
-                            reppath) or not git.is_git(reppath)):
-                        yield tag
-                    else:
-                        yield git.getBranch(reppath) or tag
-                except GitError:
-                    yield tag
-
-        return list(generateBranch())

pym/builder/variables/linux.py

@@ -67,12 +67,14 @@ class DataVarsBuilderLinux(linux.LinuxDataVars):
                     env.VariableClPkgdir(),
                     env.VariableClDistdir(),
                     env.VariableClEmergeConfig(systemRoot=self.systemRoot),
+                    env.VariableClFeatures(),
                     update.VariableClUpdateRepData(section="update"),
                     update.VariableClUpdateRepPath(section="update"),
                     update.VariableClUpdateRepName(section="update"),
                     update.VariableClUpdateRepUrl(section="update"),
                     update.VariableClUpdateRep(section="update"),
                     update.VariableClUpdateRepRev(section="update"),
+                    update.VariableClUpdateBranch(section="update"),
                     update.VariableClUpdateBranchName(section="update"),
                     update.VariableClUpdateLaymanConfig(section="update"),
                     update.VariableClUpdateLaymanStorage(section="update"),
@@ -85,8 +87,8 @@ class DataVarsBuilderLinux(linux.LinuxDataVars):
                     update.VariableClUpdateBinhostTime(section="update"),
                     update.VariableClUpdateBinhostTimeout(section="update"),
                     update.VariableClUpdateBinhosts(section="update"),
-                    update.VariableClUpdateRepoTagSet(section="update"),
-                    update.VariableClUpdateRevisionPath(section="update"),
+                    update.VariableClUpdateBinhostTimestampPath(section="update"),
+                    update.VariableClUpdateBinhostRevisionPath(section="update"),
                     system.VariableOsArchMachineGentoo()]
 
     @classmethod
@@ -112,6 +114,16 @@ class DataVarsBuilderLinux(linux.LinuxDataVars):
     def __repr__(self):
         return "Builder variables"
 
+class VariableClBuilderBinarySet(ReadonlyVariable):
+    """
+    Собираемый дистрибутив - бинарный FEATURES="getbinpkg"
+    """
+    def get(self):
+        dv = self.Get('cl_builder_linux_datavars')
+        if dv:
+            if "getbinpkg" in dv.Get('cl_features'):
+                return "on"
+        return "off"
 
 class VariableClBuilderLinuxDatavars(ReadonlyVariable):
     """
@@ -130,6 +142,8 @@ class VariableClBuilderLinuxDatavars(ReadonlyVariable):
                 with ignore(DistributiveError):
                     distr_dn = image.getDirectory()
                     dvbl = DataVarsBuilderLinux(systemRoot=distr_dn)
+                    dvbl.cache['cl_update_binhost_recheck_set'] = \
+                        self.Get('update.cl_update_binhost_recheck_set')
                     dvbl.prepare_all()
                     return dvbl
         else:

pym/builder/wsdl_builder.py

@@ -154,22 +154,26 @@ class Wsdl(WsdlBase):
                         'cl_dispatch_conf': 'usenew'},
             # описание груп (список лямбда функций)
             'groups': [
-                lambda group: group(_("Update the assemble"),
-                                    normal=('cl_builder_prepared_id',),
-                                    expert=('update.cl_update_pretend_set',
-                                            'update.cl_update_sync_only_set',
-                                            'update.cl_update_emergelist_set',
-                                            'update.cl_update_world',
-                                            'update.cl_update_skip_rb_set',
-                                            'update.cl_update_other_set',
-                                            'update.cl_update_egencache_force',
-                                            'update.cl_update_eixupdate_force',
-                                            'update.cl_update_cleanpkg_set',
-                                            'cl_builder_branch',
-                                            'cl_builder_sync_rep',
-                                            'cl_templates_locate',
-                                            'cl_verbose_set', 'cl_dispatch_conf'),
-                                    next_label=_("Perform"))]
+                lambda group: group(
+                    _("Update the assemble"),
+                    normal=('cl_builder_prepared_id',),
+                    expert=(
+                        'update.cl_update_pretend_set',
+                        'update.cl_update_sync_only_set',
+                        'update.cl_update_emergelist_set',
+                        'update.cl_update_world',
+                        'update.cl_update_skip_rb_set',
+                        'update.cl_update_other_set',
+                        'update.cl_update_egencache_force',
+                        'update.cl_update_eixupdate_force',
+                        'update.cl_update_cleanpkg_set',
+                        'update.cl_update_binhost_recheck_set',
+                        'cl_builder_rebuild_changed_set',
+                        'cl_builder_branch_data',
+                        'cl_builder_sync_rep',
+                        'cl_templates_locate',
+                        'cl_verbose_set', 'cl_dispatch_conf'),
+                    next_label=_("Perform"))]
         },
         {
             # идентификатор метода
@@ -237,7 +241,12 @@ class Wsdl(WsdlBase):
                 lambda group: group(_("Create the image"),
                                     normal=('cl_builder_prepared_id',
                                             'cl_builder_image_filename',
-                                            'cl_builder_videodrv_set'),
+                                            'cl_builder_videodrv_set',
+                                            'cl_builder_compress',
+                                            'cl_builder_isohybrid_set',
+                                            'cl_builder_keep_tree_set',
+                                            'cl_builder_prelink_set'
+                                            ),
                                     brief=('cl_builder_kernel_ver',
                                            'cl_builder_initrd_install',
                                            'os_builder_linux_shortname',
@@ -287,8 +296,7 @@ class Wsdl(WsdlBase):
                                       'update.cl_update_profile_sync_set'),
                                 normal=('cl_builder_profile_url',
                                         'cl_builder_prepared_id'),
-                                expert=('update.cl_update_profile_sync_set',
-                                        'cl_builder_profile_branch')),
+                                expert=('update.cl_update_profile_sync_set',)),
                 lambda group: group(_("Profile"),
                                 normal=('cl_builder_profile_system_shortname',
                                         'update.cl_update_world'),