From 22703407d208955c4d1d69c52ef91f7afcf962d3 Mon Sep 17 00:00:00 2001 From: Mike Khiretskiy Date: Mon, 25 May 2015 09:54:47 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D0=BF=D0=B5=D1=80=D0=B5=D0=BC=D0=B5=D0=BD=D0=BD?= =?UTF-8?q?=D1=8B=D0=B5=20=D0=B4=D0=BB=D1=8F=20=D0=B2=D1=8B=D0=B1=D0=BE?= =?UTF-8?q?=D1=80=D0=B0=20=D0=BF=D1=80=D0=BE=D1=84=D0=B8=D0=BB=D1=8F.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pym/builder/build_storage.py | 6 +- pym/builder/builder.py | 24 ++ pym/builder/utils/cl_builder_image.py | 77 +++++ pym/builder/utils/cl_builder_restore.py | 11 +- pym/builder/variables/__init__.py | 1 + pym/builder/variables/action.py | 3 +- pym/builder/variables/builder.py | 433 +++++++++++++++++++++--- pym/builder/variables/linux.py | 101 ++++-- pym/builder/variables/profiles.py | 330 ++++++++++++++++++ pym/builder/wsdl_builder.py | 55 +-- 10 files changed, 941 insertions(+), 100 deletions(-) create mode 100644 pym/builder/utils/cl_builder_image.py create mode 100644 pym/builder/variables/profiles.py diff --git a/pym/builder/build_storage.py b/pym/builder/build_storage.py index 4802334..f32d86d 100644 --- a/pym/builder/build_storage.py +++ b/pym/builder/build_storage.py @@ -97,6 +97,9 @@ class Build(object): """ Закрыть дистрибутив привязанный к сборке """ + self.distributive.release() + for child in self.distributive.childs: + child.release() self.distributive.close() def restore(self): @@ -231,7 +234,8 @@ class BuildStorage(object): _("Failed to get distributive information")) dist = Distributive.unserialize(data) build = Build(build_id, dist, self, restore=True) - if isMount(build_dir): + proc_build_dir = path.join(build_dir, 'proc') + if isMount(build_dir) and isMount(proc_build_dir): build.status = build.Status.WORKED else: build.status = build.Status.BROKEN diff --git a/pym/builder/builder.py b/pym/builder/builder.py index 002e521..f9aa32c 100644 --- a/pym/builder/builder.py +++ b/pym/builder/builder.py @@ -16,6 +16,7 @@ import sys from .build_storage import Build from calculate.install.distr import Distributive +from os import path from .datavars import BuilderError from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, _) @@ -35,6 +36,11 @@ class Builder(object): dir_distro.mountSystemDirectories() return True + def umount_system(self, target): + dir_distro = target.convertToDirectory() + dir_distro.umountSystemDirectories() + return True + def detach_target(self, target): """ @type target: Distributive @@ -63,3 +69,21 @@ class Builder(object): build.pkgdir = self.clVars.Get('cl_builder_pkgdir') build.save() return True + + def set_builder_action(self, action_name): + self.clVars.Set('cl_builder_action', action_name, force=True) + return True + + def prepare_iso(self, dn): + self.endTask() + self.startTask(_("Prepare iso data")) + root_path = path.relpath(dn, self.clVars.Get('cl_builder_path')) + self.applyTemplates(self.clVars.Get('cl_builder_target'), + False, False, root_path) + self.endTask() + self.startTask(_("Pack iso image")) + self.addProgress() + + def prepare_image(self, image): + image.eventPrepareIso.connect(self.prepare_iso) + return True diff --git a/pym/builder/utils/cl_builder_image.py b/pym/builder/utils/cl_builder_image.py new file mode 100644 index 0000000..411ad97 --- /dev/null +++ b/pym/builder/utils/cl_builder_image.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Calculate Ltd. http://www.calculate-linux.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from calculate.core.server.func import Action, Tasks +from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate +from calculate.lib.utils.files import FilesError +from ..datavars import BuilderError +from calculate.lib.utils.portage import GitError +from calculate.install.distr import DistributiveError, IsoDistributive + +setLocalTranslate('cl_builder3', sys.modules[__name__]) +__ = getLazyLocalTranslate(_) + + +class ClBuilderImageAction(Action): + """ + Действие обновление конфигурационных файлов + """ + # ошибки, которые отображаются без подробностей + native_error = (DistributiveError, FilesError, BuilderError, GitError) + + successMessage = __("Image created successfully") + failedMessage = __("Creating the image is failed") + interruptMessage = __("Image creating manually interrupted") + + # список задач для действия + tasks = [ + # закрепить подключенные данные + {'name': 'detach', + 'message': __("Detach target"), + 'method': 'Builder.detach_target(cl_builder_target)', + }, + {'name': 'umount_system', + 'message': __("Umount build system resources"), + 'method': 'Builder.umount_system(cl_builder_target)', + 'condition': lambda Get: Get('cl_builder_build') + }, + {'name': 'prepare_image', + 'method': 'Builder.prepare_image(cl_builder_image)', + 'condition': lambda Get: isinstance(Get('cl_builder_image'), + IsoDistributive) + }, + {'name': 'squash_action', + 'method': 'Builder.set_builder_action("squash")', + }, + {'name': 'apply_template', + 'message': __("Configuring squash"), + # наложить шаблоны в развернутый дистрибутив + 'method': 'Builder.applyTemplates(cl_builder_target,False,False,None)', + }, + {'name': 'squash_action', + 'method': 'Builder.set_builder_action("iso")', + }, + # распаковка дистрибутива + {'name': 'unpack', + 'message': __("Pack squash"), + 'method': 'Install.unpack(cl_builder_target, cl_builder_image, "0")', + }, + {'name': 'umount_system!:mount_system', + 'warning': _("Restore build system resources"), + 'method': 'Builder.mount_target(cl_builder_target)', + 'condition': lambda Get: Get('cl_builder_build') + }] diff --git a/pym/builder/utils/cl_builder_restore.py b/pym/builder/utils/cl_builder_restore.py index 87eef68..22a55e0 100644 --- a/pym/builder/utils/cl_builder_restore.py +++ b/pym/builder/utils/cl_builder_restore.py @@ -17,7 +17,7 @@ import sys from calculate.core.server.func import Action from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate -from calculate.lib.utils.files import FilesError +from calculate.lib.utils.files import FilesError, isMount from ..datavars import BuilderError from calculate.lib.utils.portage import GitError from calculate.install.distr import DistributiveError @@ -33,15 +33,16 @@ class ClBuilderRestoreAction(Action): # ошибки, которые отображаются без подробностей native_error = (DistributiveError, FilesError, BuilderError, GitError) - successMessage = __("Assemble broken successfully") - failedMessage = __("Breaking the assemble is failed") - interruptMessage = __("Breaking manually interrupted") + successMessage = __("Build was restored successfully") + failedMessage = __("Restoring the build is failed") + interruptMessage = __("Restoring manually interrupted") # список задач для действия tasks = [ {'name': 'restore_build', 'message': __("Restoring {cl_builder_id} assemble"), - 'method': 'Builder.restore_build(cl_builder_build)' + 'method': 'Builder.restore_build(cl_builder_build)', + 'condition': lambda Get: not isMount(Get('cl_builder_path')) }, # подмонтировать необходимые данные {'name': 'mount', diff --git a/pym/builder/variables/__init__.py b/pym/builder/variables/__init__.py index 921bdb1..eeeac73 100644 --- a/pym/builder/variables/__init__.py +++ b/pym/builder/variables/__init__.py @@ -17,5 +17,6 @@ import action import builder import linux +import profiles section = "builder" diff --git a/pym/builder/variables/action.py b/pym/builder/variables/action.py index 17b1fe2..06a14de 100644 --- a/pym/builder/variables/action.py +++ b/pym/builder/variables/action.py @@ -18,7 +18,8 @@ import sys from calculate.lib.datavars import Variable,VariableError,ReadonlyVariable from calculate.lib.cl_lang import setLocalTranslate -setLocalTranslate('cl_builder3',sys.modules[__name__]) + +setLocalTranslate('cl_builder3', sys.modules[__name__]) class VariableAcBuilderSquash(ReadonlyVariable): diff --git a/pym/builder/variables/builder.py b/pym/builder/variables/builder.py index b5a6426..4ea0083 100644 --- a/pym/builder/variables/builder.py +++ b/pym/builder/variables/builder.py @@ -16,18 +16,47 @@ import sys from os import path -from calculate.install.distr import (Distributive, DistributiveError, - LayeredDistributive, DefaultMountPath, PartitionDistributive, - DirectoryDistributive) +import os +import re +from calculate.install import distr from calculate.lib.utils.device import getUdevDeviceInfo, humanreadableSize -from calculate.lib.utils.files import isMount, process +from calculate.lib.utils.files import isMount, process, typeFile, listDirectory, \ + pathJoin +from calculate.lib.utils.kernel import InitrdFile +from calculate.lib.utils.tools import max_default from ..build_storage import BuildStorage, Build from ..drive_spool import DriveSpool -from calculate.lib.datavars import Variable,VariableError,ReadonlyVariable +from calculate.lib.datavars import Variable, VariableError, ReadonlyVariable +from functools import wraps from calculate.lib.cl_lang import setLocalTranslate setLocalTranslate('cl_builder3',sys.modules[__name__]) +def debug(func): + @wraps(func) + def _wrapped_func(*args, **kw): + ret = func(*args, **kw) + print "MYDEBUG",ret + return ret + return _wrapped_func + +def is_action(*available_action, **action_kwargs): + def decorator(func): + @wraps(func) + def _wrapped_func(self, *args, **kw): + if self.Get('cl_action') in available_action: + return func(self, *args, **kw) + else: + return action_kwargs.get('default_value', '') + return _wrapped_func + return decorator + +def as_list(func): + @wraps(func) + def _wrapped_func(self, *args, **kw): + return list(func(self, *args, **kw)) + return _wrapped_func + class VariableClBuilderStorage(ReadonlyVariable): type = "object" @@ -70,6 +99,8 @@ class VariableClBuilderSourceFilename(Variable): """ Названия файла исходного дистрибутива """ + type = "file" + element = 'file' opt = ["--source"] value = "" metavalue = "SOURCE" @@ -82,22 +113,24 @@ class VariableClBuilderSourceFilename(Variable): """Set image file""" if self.Get('cl_action') == 'create' and not isoimage: raise VariableError(_("You need to select a source image")) - imageData = Distributive().getInfo(isoimage) - if not("os_linux_shortname" in imageData and - imageData.get('os_linux_build','') and - "os_arch_machine" in imageData): + shortname = self.Get('os_builder_linux_shortname') + build = self.Get('os_builder_linux_build') + arch = self.Get('os_builder_arch_machine') + if not build or not shortname or not arch: raise VariableError(_("Wrong image file")) def humanReadable(self): fullname = self.Get('os_builder_linux_name') subname = self.Get('os_builder_linux_subname') + ver = self.Get('os_builder_linux_ver') if subname: subname = " %s" % subname + if ver: + ver = " %s"%ver arch = self.Get('os_builder_arch_machine') build = self.Get('os_builder_linux_build') - ver = self.Get('os_builder_linux_ver') - return "{fullname} {ver} {arch} {build}".format( + return "{fullname}{ver} {arch} {build}".format( fullname="%s%s" % (fullname, subname), build=build, ver=ver, arch=arch) @@ -113,8 +146,8 @@ class VariableClBuilderSource(ReadonlyVariable): try: filename = self.Get('cl_builder_source_filename') if filename: - return Distributive.fromFile(filename) - except DistributiveError: + return distr.Distributive.fromFile(filename) + except distr.DistributiveError: pass return "" @@ -139,7 +172,7 @@ class VariableClBuilderDiskDev(Variable): def get(self): ds = self.Get('cl_builder_device_spool') - return ds.get().next() + return ds.get() or "" def check(self, value): if value.startswith("/dev"): @@ -189,12 +222,13 @@ class VariableClBuilderDiskSize(ReadonlyVariable): action = self.Get('cl_action') if action == 'create': return self.get_create() - elif action in ('break', 'repair'): + elif action in ('break', 'restore'): return def humanReadable(self): return humanreadableSize(int(self.Get())) + class VariableClBuilderLayeredSet(Variable): """ Сборка будет выполняться в слое @@ -213,6 +247,9 @@ class VariableClBuilderLayeredSet(Variable): if self.Get('cl_builder_disk_dev').startswith('/dev'): raise VariableError( _("Layers are used for building in a directory only")) + if isinstance(self.Get('cl_builder_source'), distr.ArchiveDistributive): + raise VariableError( + _("Layers are used for building from iso image")) class VariableClBuilderPath(ReadonlyVariable): @@ -236,32 +273,35 @@ class VariableClBuilderTarget(Variable): builder_disk_dev = self.Get('cl_builder_disk_dev') if not builder_disk_dev: return "" - buildid = self.Get('cl_builder_id') - mountdir = path.join(DefaultMountPath.BaseMountPath, buildid) + build_id = self.Get('cl_builder_id') + mount_dir = path.join(distr.DefaultMountPath.BaseMountPath, build_id) source = self.Get('cl_builder_source_filename') if self.GetBool('cl_builder_layered_set'): - distr = LayeredDistributive(mountdir, builder_disk_dev, source) - return distr + dist_obj = distr.LayeredDistributive(mount_dir, builder_disk_dev, + source) + return dist_obj else: if builder_disk_dev.startswith('/dev'): - distr = PartitionDistributive(builder_disk_dev, - mdirectory=mountdir) - return distr + dist_obj = distr.PartitionDistributive(builder_disk_dev, + mdirectory=mount_dir) + return dist_obj else: - distr = DirectoryDistributive(builder_disk_dev, - mdirectory=mountdir) - return distr + dist_obj = distr.DirectoryDistributive(builder_disk_dev, + mdirectory=mount_dir) + return dist_obj def get_worked(self): build = self.Get('cl_builder_build') if build: - return build.distributive + dist_obj = build.distributive + return dist_obj return "" def get(self): - if self.Get('cl_action') == 'create': + action = self.Get('cl_action') + if action == 'create': return self.get_create() - elif self.Get('cl_action') in ('break', 'restore'): + elif action in ('break', 'restore', 'image'): return self.get_worked() else: return "" @@ -299,12 +339,17 @@ class VariableClBuilderBuild(Variable): storage = self.Get('cl_builder_storage') buildid = self.Get('cl_builder_id') build = storage.get_build(buildid) + if build and build.distributive: + build.distributive.reserve() + for child in build.distributive.childs: + child.reserve() return build or "" def get(self): - if self.Get('cl_action') == 'create': + action = self.Get('cl_action') + if action == 'create': return self.get_create() - if self.Get('cl_action') in ('break', 'restore'): + if action in ('break', 'restore', 'image'): return self.get_worked() return "" @@ -330,32 +375,33 @@ class VariableClBuilderPreparedId(BaseBuildId): type = "choice" untrusted = True + @as_list def available(self): bs = self.Get('cl_builder_storage') action = self.Get('cl_action') for x in bs: build = bs.get_build(x) - if action == 'break' or build.status == Build.Status.WORKED: + if (build and (action == 'break' or + build.status == Build.Status.WORKED)): yield x def get(self): - l = list(self.available()) + l = self.available() if l and len(l) == 1: return l[0] return "" + @as_list def choice(self): - def generator(): - bs = self.Get('cl_builder_storage') - for x in bs: - build = bs.get_build(x) + bs = self.Get('cl_builder_storage') + for x in bs: + build = bs.get_build(x) + if build: if build.status == Build.Status.BROKEN: yield (x, "%s (%s)" % (x, _("broken"))) else: yield (x, x) - return list(generator()) - def check(self, value): if not value: raise VariableError(_("Please select the assemble id")) @@ -375,33 +421,34 @@ class VariableClBuilderBrokenId(BaseBuildId): type = "choice" untrusted = True + @as_list def available(self): bs = self.Get('cl_builder_storage') for x in bs: build = bs.get_build(x) - if build.status == Build.Status.BROKEN: + if build and build.status == Build.Status.BROKEN: yield x + @is_action('restore') def get(self): - l = list(self.available()) + l = self.available() if l and len(l) == 1: return l[0] return "" + @as_list + @is_action('restore', default_value=[]) def choice(self): - def generator(): - bs = self.Get('cl_builder_storage') - for x in bs: - build = bs.get_build(x) - if build.status == Build.Status.BROKEN: - yield (x, "%s (%s)" % (x, _("broken"))) - - return list(generator()) + bs = self.Get('cl_builder_storage') + for x in bs: + build = bs.get_build(x) + if build.status == Build.Status.BROKEN: + yield (x, "%s (%s)" % (x, _("broken"))) def check(self, value): if not value: raise VariableError(_("Please select the assemble id")) - l = list(self.available()) + l = self.available() if not l: raise VariableError(_("Assemble %s is not found") % value) @@ -414,7 +461,7 @@ class VariableClBuilderId(ReadonlyVariable): action = self.Get('cl_action') if action == 'create': return self.Get('cl_builder_new_id') - elif action == 'break': + elif action in ('break', 'image'): return self.Get('cl_builder_prepared_id') elif action == 'restore': return self.Get('cl_builder_broken_id') @@ -466,7 +513,7 @@ class VariableClBuilderPkgdir(Variable): action = self.Get('cl_action') if action == 'create': return self.fallback() - elif action in ('restore',): + elif action in ('restore', 'image'): build = self.Get('cl_builder_build') return build.pkgdir or self.fallback() else: @@ -478,3 +525,287 @@ class VariableClBuilderAction(ReadonlyVariable): Дополнительное действие по созданию образа: iso, squash. """ value = "" + + +class VariableClBuilderImageFilename(ReadonlyVariable): + """ + Название iso образа + """ + opt = ["--iso"] + value = "" + metavalue = "IMAGE" + + def init(self): + self.label = _("Image name") + self.help = _("set image name") + + def get(self): + base_dn = self.Get('cl_builder_base_path') + build_id = self.Get('cl_builder_id') + if build_id: + shortname = self.Get('os_builder_linux_shortname') + buildnumber = self.Get('os_builder_linux_build') + arch = self.Get('os_builder_arch_machine') + imagename = "%s-%s-%s.iso" % (shortname, buildnumber, + arch) + return path.join(base_dn, build_id, "linux", imagename) + return "" + + +class VariableClBuilderIsoBasePath(Variable): + """ + Базовый путь, где будут подготавливаться данные, которые будут запакованы в iso + """ + value = "/var/calculate/tmp" + + +class VariableClBuilderIsoPath(ReadonlyVariable): + """ + Путь, где будут подготавливаться данные, которые будут запакованы в iso + """ + def get(self): + base_dn = self.Get('cl_builder_iso_base_path') + build_id = self.Get('cl_builder_id') + if build_id: + dn = "iso-%s" % self.Get('cl_builder_id') + directory = path.join(base_dn, dn) + new_dn = directory + for i in range(0, 9999): + if not path.exists(new_dn): + return new_dn + else: + new_dn = "%s.%04d" % (directory, i) + return new_dn + return "" + + +class VariableClBuilderSquashPath(ReadonlyVariable): + """ + Путь от iso до содержимого squash + """ + @is_action('image') + def get(self): + return path.relpath(self.Get('cl_builder_path'), + self.Get('cl_builder_iso_path')) + + +class VariableClBuilderImage(ReadonlyVariable): + """ + Создаваемый образ + """ + @is_action('image') + def get(self): + image_name = self.Get('cl_builder_image_filename') + bdn = self.Get('cl_builder_iso_path') + iso = distr.IsoDistributive(image_name, bdirectory=bdn, + vol_id=self.Get('cl_builder_iso_label')) + return iso + + +class VariableClBuilderLiveSet(Variable): + """ + Вызывать только live шаблоны при первой загрузке + """ + type = "bool" + value = "on" + + +class VariableClBuilderCdname(ReadonlyVariable): + """ + Type of iso (CD/DVD) + """ + @is_action("image") + def get(self): + squashfile = pathJoin(self.Get('cl_builder_iso_path'), + self.Get('cl_builder_current_squash')) + kernelfile = pathJoin(self.Get('cl_builder_iso_path'), + self.Get('cl_builder_squash_path'), + 'boot', + self.Get('cl_builder_kernel')) + initrdfile = pathJoin(self.Get('cl_builder_iso_path'), + self.Get('cl_builder_squash_path'), + 'boot', + self.Get('cl_builder_initrd_install')) + if os.access(squashfile, os.R_OK) and os.access(kernelfile, os.R_OK) and \ + os.access(initrdfile, os.R_OK): + isosize = path.getsize(squashfile) + path.getsize(kernelfile) + \ + path.getsize(initrdfile) + 2 * 1024 * 1024 + if isosize > 700 * 1024 * 1024: + return "DVD" + else: + return "CD" + return "" + + +class VariableClBuilderIsoLabel(Variable): + """ + LABEL для iso + """ + @is_action("image") + def get(self): + return "%s-%s" % (self.Get('os_builder_linux_shortname').upper(), + self.Get('os_builder_linux_build')) + + +class VariableClBuilderRootParam(Variable): + """ + параметр root= для livecd + """ + + @is_action("image") + def get(self): + return "live:LABEL=%s" % self.Get('cl_builder_iso_label') + + +class VariableClBuilderCurrentSquash(ReadonlyVariable): + """ + Создаваемый livecd.squash + """ + value = "livecd.squashfs" + + +class VariableClBuilderKernelCmd(ReadonlyVariable): + """ + Параметры по умолчанию для calcboot + """ + value = "" + + +class VariableClBuilderKernelVer(ReadonlyVariable): + """ + Текущая версия ядра + """ + def init(self): + self.label = _("Kernel version") + + def get(self): + prefix = self.Get('cl_builder_path') + if prefix: + current_src = self.get_current_kernel_src(prefix) + src = path.join(prefix,current_src) + return self.get_src_kernel_version(src) + + def get_config_version(self, configfile): + re_config = re.compile("Automatically generated file;.*\n" + ".*?Linux/\S+\s+(\S+)\s", re.M) + if path.exists(configfile): + with open(configfile) as f: + match = re_config.search(f.read(200)) + if match: + return match.group(1) + + def get_src_kernel_version(self, kernel_src): + """ + Get version of kernel from .config + """ + config_path = path.join(kernel_src, ".config") + makefile_path = path.join(kernel_src, "Makefile") + + # get version from config + version = self.get_config_version(config_path) + if version: + return version + + # get version from Makefile + re_makefile = re.compile("^VERSION = (\S+)\n" + "PATCHLEVEL = (\S+)\n" + "SUBLEVEL = (\S+)\n" + "EXTRAVERSION = (\S*)\n", re.M) + if path.exists(makefile_path): + with open(makefile_path) as f: + match = re_makefile.search(f.read(200)) + if match: + return "{0}.{1}.{2}{3}".format(*match.groups()) + return "" + + def get_current_kernel_src(self, prefix): + src_path = "usr/src" + current_linux_src = path.join(src_path, "linux") + symlink_kernel = path.join(prefix, current_linux_src) + if not path.exists(symlink_kernel) or not path.islink(symlink_kernel): + raise ValueError("Failed to determine current kernel version") + return path.join(src_path, os.readlink(symlink_kernel)) + + +class KernelData(ReadonlyVariable): + """ + Данные о текущем ядре + """ + kernel_object = "kernel" + file_description = "" + + def filter(self, x, version=None): + return x + + def list(self, prefix='/', bootdir='boot'): + boot_dir = path.join(prefix, bootdir) + return self.get_files_by_type(boot_dir, self.file_description) + + def get(self): + prefix = self.Get('cl_builder_path') + version = self.Get('cl_builder_kernel_ver') + obj_file = max_default( + self.filter(self.list(prefix), version=version), + key=path.getmtime, + default="") + if obj_file: + obj_file = path.basename(obj_file) + return obj_file + + def get_files_by_type(self, pathname, descr): + ftype = typeFile(magic=0x4).getMType + for x in listDirectory(pathname, fullPath=True): + if descr in ftype(x): + yield x + + +class VariableClBuilderInitrdInstall(KernelData): + """ + Текущий initrd + """ + file_description = "ASCII cpio archive" + + def init(self): + self.label = _("Init RAM fs") + + def filter(self, iterable, version=None): + for fn in iterable: + if InitrdFile(fn).get_kernel_version() == version: + yield fn + + +class VariableClBuilderKernel(KernelData): + """ + Текущее ядро + """ + file_description = "boot executable bzImage" + + def init(self): + self.label = _("Kernel file") + + def filter(self, iterable, version=None): + ftype = typeFile(magic=0x4).getMType + re_kver = re.compile("bzImage, version (\S+)\s") + for fn in iterable: + m = re_kver.search(ftype(fn)) + if m.group(1) == version: + yield fn + + +class VariableClBuilderTemplateLocation(Variable): + """ + Устанавливаются только дистрибутивные шаблоны + """ + def get(self): + return [x for x in self.Get('main.cl_template_location') + if x not in ('remote', 'local')] + + +class VariableClBuilderBranchName(Variable): + """ + Ветки оверлеев используемые для сборки системы + """ + type = "list" + def get(self): + # TODO: заглушка + return self.Get('update.cl_update_branch_name') diff --git a/pym/builder/variables/linux.py b/pym/builder/variables/linux.py index 21004d5..0b129e1 100644 --- a/pym/builder/variables/linux.py +++ b/pym/builder/variables/linux.py @@ -17,9 +17,18 @@ import sys import os from os import path -from calculate.lib.datavars import ReadonlyVariable +import re +from calculate.install import distr +from calculate.install.distr import DistributiveError +from calculate.lib.datavars import ReadonlyVariable, SimpleDataVars +from calculate.lib.utils.tools import ignore +from calculate.lib.variables import linux +from calculate.update.variables import update +from calculate.lib.variables import system from calculate.lib.cl_lang import setLocalTranslate +import datetime +import profiles setLocalTranslate('cl_builder3', sys.modules[__name__]) @@ -27,32 +36,80 @@ setLocalTranslate('cl_builder3', sys.modules[__name__]) class BuilderLinux(ReadonlyVariable): # variable for get current system info (example: os_linux_shortname) variable = "" - # field of distroinfo (name,ver,build,march and etc) - distroinfo_field = "" + def get(self): + dv = self.Get('cl_builder_linux_datavars') + if dv: + return dv[self.variable] + else: + return "" + +class DataVarsBuilderLinux(linux.LinuxDataVars): + """ + Упрощенная модель переменных для собираемого образа + """ + def variables(self): + l = super(DataVarsBuilderLinux, self).variables() + return l + [update.VariableClProfileSystem(), + update.VariableClProfileRepository(), + system.VariableOsArchMachineGentoo(), + update.VariableClProfileData()] + + @classmethod + def StageInformation(cls, fn): + re_stage = re.compile("stage(\d)-(amd64|x86)-(\d{8}).tar.\w+") + m = re_stage.search(fn) + if m: + map_arch = {'amd64': 'x86_64', 'x86': 'i686'} + c = {'os_linux_build': m.group(3), + 'os_arch_machine':map_arch[m.group(2)], + 'os_linux_shortname': 'Gentoo', + 'os_linux_subname': "Stage%s"%m.group(1), + 'cl_profile_name': 'Gentoo profile', + 'os_linux_name': 'Gentoo'} + else: + c = {} + obj = cls(cache=c) + for vn in obj.allVars.keys(): + if vn not in obj.cache: + obj.cache[vn] = '' + return obj + + def __repr__(self): + return "Builder variables" + + +class VariableClBuilderLinuxDatavars(ReadonlyVariable): + """ + Переменные дистрибутива + """ def source_data(self): image = self.Get('cl_builder_source') + image_fn = self.Get('cl_builder_source_filename') + return self.get_data(image, image_fn=image_fn) + + def get_data(self, image, image_fn=None): if image: - d = image.getInfo() - res = d.get(self.variable, "") - return str(res()) if callable(res) else res + if isinstance(image, distr.ArchiveDistributive): + return DataVarsBuilderLinux.StageInformation(image_fn) + with image: + with ignore(DistributiveError): + distr_dn = image.getDirectory() + dvbl = DataVarsBuilderLinux(systemRoot=distr_dn) + dvbl.prepare_all() + return dvbl else: return "" def target_data(self): image = self.Get('cl_builder_target') - if image: - d = image.getInfo() - res = d.get(self.variable, "") - return str(res()) if callable(res) else res - else: - return "" + return self.get_data(image, image_fn=None) def get(self): """Get by distroinfo or current info""" if self.Get('cl_action') == 'create': return self.source_data() - elif self.Get('cl_action') in ('break', 'restore'): + elif self.Get('cl_action') in ('break', 'restore', 'image'): return self.target_data() return "" @@ -73,6 +130,14 @@ class VariableOsBuilderLinuxBuild(BuilderLinux): """Shortname of system""" variable = "os_linux_build" + def get(self): + action = self.Get('cl_action') + if action == 'image': + curdate = datetime.datetime.now() + return "%04d%02d%02d" % (curdate.year, curdate.month, curdate.day) + else: + return BuilderLinux.get(self) + class VariableOsBuilderArchMachine(BuilderLinux): """Shortname of system""" @@ -103,13 +168,3 @@ class VariableOsBuilderLinuxSubname(BuilderLinux): Install subname """ variable = "os_linux_subname" - - -class VariableClBuilderProfileName(BuilderLinux): - """ - Install subname - """ - def init(self): - self.label = _("System profile") - - variable = "cl_profile_name" diff --git a/pym/builder/variables/profiles.py b/pym/builder/variables/profiles.py new file mode 100644 index 0000000..3a47ed5 --- /dev/null +++ b/pym/builder/variables/profiles.py @@ -0,0 +1,330 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Calculate Ltd. http://www.calculate-linux.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +from os import path +from calculate.lib.datavars import (ReadonlyVariable, FieldValue, VariableError, + DataVarsError, Variable) +from calculate.lib.utils.portage import GitError +from calculate.lib.variables import linux as lib_linux +from calculate.install import distr +from calculate.update.variables import update +from calculate.update import profile as update_profile + + +from calculate.lib.cl_lang import setLocalTranslate + +setLocalTranslate('cl_builder3', sys.modules[__name__]) + +class VariableClBuilderProfileStorage(ReadonlyVariable): + type = "object" + + def get(self): + return update_profile.RepositoryStorageSet( + update_profile.CacheStorage('/var/calculate/tmp/update')) + + +class VariableClBuilderProfileUrl(update.VariableClUpdateProfileUrl): + """ + URL репозитория + """ + check_after = ["cl_builder_profile_branch"] + + def check(self, value): + action = self.Get('cl_action') + if not value and action == 'create': + branch = self.Get('cl_builder_profile_branch') + if branch != update_profile.DEFAULT_BRANCH: + raise VariableError("Need to specify profile repository") + return + if not value: + raise VariableError("Need to specify profile repository") + try: + branch = self.Get('cl_builder_profile_branch') + self.Get('cl_builder_profile_storage').get_profiles(value, branch) + except GitError as e: + raise VariableError(str(e)) + if not self.Get('cl_builder_profile_shortname'): + raise VariableError(_("Repository %s has no profiles")%value) + + def get(self): + action = self.Get('cl_action') + if action == 'create': + return "" + if action in ("image"): + return update.VariableClUpdateProfileUrl.get(self) + return "" + + +class VariableClBuilderProfileBranch(update.VariableClUpdateProfileBranch): + """ + Текущая ветка репозитория + """ + + def get(self): + action = self.Get('cl_action') + if action in ("image"): + pass + #rep_set = self.Get('cl_update_profile_storage') + #url = self.Get('cl_update_profile_url') + ##print url, rep_set.is_local(url, branch=None) + #if rep_set.is_local(url, branch=None): + # rep = rep_set.get_repository(url, branch=None) + # git = Git() + # return git.getBranch(rep.directory) + return update_profile.DEFAULT_BRANCH + +class VariableClProfileCacheRepository(ReadonlyVariable): + """ + Репозиторий из которого будет извлечён список профилей + """ + type = "object" + + def get(self): + url = self.Get('cl_builder_profile_url') + if not url: + return "" + try: + rep_set = self.Get('cl_builder_profile_storage') + branch = self.Get('cl_builder_profile_branch') + rep = rep_set.get_repository(url, branch) + #if rep and self.Get('cl_update_profile_sync_set') == 'on': + if rep: + rep.sync() + return rep_set.get_repository(url, branch) or "" + except GitError: + return "" + + +class VariableClProfileCacheData(update.VariableClProfileData): + source = [] + + repository = "cl_profile_cache_repository" + +class VariableClBuilderProfileData(update.VariableClProfileData): + type = "table" + source = ["cl_builder_profile_fullname", + "cl_builder_profile_shortname", + "cl_builder_profile_path", + "cl_builder_profile_arch"] + + def init(self): + self.label = "test" + + def get(self, hr=False): + url = self.Get('cl_builder_profile_url') + if not url: + dv = self.Get('cl_builder_linux_datavars') + if dv: + return dv['cl_profile_data'] + return [[]] + else: + return self.Get('cl_profile_cache_data') + + +class VariableClBuilderProfileFullname(FieldValue, ReadonlyVariable): + """ + Полное название профиля + """ + type = "list" + source_variable = "cl_builder_profile_data" + column = 0 + + +class VariableClBuilderProfileShortname(FieldValue, ReadonlyVariable): + """ + Полное название профиля + """ + type = "list" + source_variable = "cl_builder_profile_data" + column = 1 + + +class VariableClBuilderProfilePath(FieldValue, ReadonlyVariable): + """ + Полное название профиля + """ + type = "list" + source_variable = "cl_builder_profile_data" + column = 2 + + +class VariableClBuilderProfileArch(FieldValue, ReadonlyVariable): + """ + Полное название профиля + """ + type = "list" + source_variable = "cl_builder_profile_data" + column = 3 + + +class VariableClBuilderProfileSystemShortname(Variable): + """ + Профиль системы + """ + type = "choice" + opt = ["--profile"] + untrusted = True + metavalue = "PROFILE" + + def init(self): + self.label = _("System profile") + self.help = _("set the system profile") + + def check_profile(self, path_profile, profile_fullname): + if path_profile: + dv = update.DataVarsUpdateProfile(path_profile) + if ":" in profile_fullname: + repo_name = profile_fullname.partition(":")[0] + else: + repo_name = "" + try: + if (not dv.Get('cl_update_rep_name') or + not dv.Get('cl_update_rep_url')): + raise VariableError(_("Repository variables " + "were not configured for the profile")) + if not dv.Get('os_linux_name'): + raise VariableError("") + if repo_name not in list(dv.Get('cl_update_rep_name')): + raise VariableError( + _("Overlay %s is not specified in cl_update_rep_name")% + repo_name) + except (DataVarsError, VariableError) as e: + if str(e): + message = ". " + str(e) + else: + message = "" + raise VariableError(_("The selected profile is not Calculate") + + message) + else: + raise VariableError(_("Wrong Calculate profile")) + + def check(self, profile): + if not profile: + raise VariableError(_("You must specify the profile")) + action = self.Get('cl_action') + if action == 'create': + url = self.Get('cl_builder_profile_url') + # внеший репозиторий профилей + if url: + path_profile, profile_fullname = \ + path_profile = self.Select( + 'cl_builder_profile_path', + where='cl_builder_profile_shortname', + eq=profile, limit=1) + profile_fullname = self.Select( + 'cl_builder_profile_fullname', + where='cl_builder_profile_shortname', + eq=profile, limit=1) + self.check_profile(path_profile, profile_fullname) + # репозиторий в образе + else: + self.check_inner_profile(profile) + + def check_inner_profile(self, profile): + """ + Проверка профиля находящегося в репозитории образа системы + """ + image = self.Get('cl_builder_source') + dv = self.Get('cl_builder_linux_datavars') + if not image or not dv: + raise VariableError(_("Failed to check profile: " + "wrong source")) + if image: + if isinstance(image, distr.ArchiveDistributive): + raise VariableError(_("Unavailable to use profile from " + "stage distributive")) + with image: + try: + distr_dn = image.getDirectory() + profile_dn = self.Select( + 'cl_builder_profile_path', + where="cl_builder_profile_shortname", + eq=profile, limit=1) + profile_fullname = self.Select( + 'cl_builder_profile_fullname', + where="cl_builder_profile_shortname", + eq=profile, limit=1) + if not profile_dn: + raise VariableError() + profile_dn = dv.change_root(profile_dn, distr_dn) + self.check_profile(profile_dn, profile_fullname) + except distr.DistributiveError: + raise VariableError(_("Failed to check profile: " + "failed to mount source")) + + def get_by_url(self): + shortname = self.Get('cl_builder_profile_shortname') + if len(shortname) == 1: + return shortname[0] + return "" + + def get_from_image(self): + dv = self.Get('cl_builder_linux_datavars') + if dv: + try: + profile_system = dv["cl_profile_system"] + for (fullname, shortname, + _path, arch) in dv.Get('cl_profile_data'): + if _path == profile_system: + return shortname + except VariableError: + pass + return "" + + def get(self): + action = self.Get('cl_action') + if action == 'create': + url = self.Get('cl_builder_profile_url') + if url: + return self.get_by_url() + else: + return self.get_from_image() + return "" + + def choice(self): + arch = self.Get('os_arch_machine_gentoo') + profiles = zip(*self.Select(['cl_builder_profile_shortname', + 'cl_builder_profile_fullname'], + where='cl_builder_profile_arch', eq=arch)) + if profiles: + short_name, full_name = profiles + return zip(short_name, full_name) + return [] + + +class VariableClBuilderProfileSystem(ReadonlyVariable): + def get(self): + action = self.Get('cl_action') + if action == 'create': + shortname = self.Get('cl_builder_profile_system_shortname') + profile = self.Select('cl_builder_profile_path', + where="cl_builder_profile_shortname", + eq=shortname, limit=1) + if profile: + return profile + return "" + + +class VariableClBuilderProfileName(lib_linux.VariableClProfileName): + """ + Install subname + """ + source_var = "cl_builder_profile_system" + + def init(self): + self.label = _("System profile") diff --git a/pym/builder/wsdl_builder.py b/pym/builder/wsdl_builder.py index 6d695b2..77ae082 100644 --- a/pym/builder/wsdl_builder.py +++ b/pym/builder/wsdl_builder.py @@ -26,6 +26,7 @@ from calculate.lib.utils.portage import GitError from utils.cl_builder_create import ClBuilderCreateAction from utils.cl_builder_break import ClBuilderBreakAction from utils.cl_builder_restore import ClBuilderRestoreAction +from utils.cl_builder_image import ClBuilderImageAction from calculate.lib.cl_lang import setLocalTranslate,getLazyLocalTranslate setLocalTranslate('cl_builder3',sys.modules[__name__]) __ = getLazyLocalTranslate(_) @@ -66,20 +67,25 @@ class Wsdl(WsdlBase): # описание груп (список лямбда функций) 'groups': [ lambda group: group(_("Create the system"), - normal=('cl_builder_source_filename', - 'cl_builder_new_id', - 'cl_builder_disk_dev', - 'cl_builder_layered_set'), - expert=('cl_templates_locate', - 'cl_verbose_set', - 'cl_dispatch_conf'), - hide=('cl_templates_locate', - 'cl_verbose_set', - 'cl_dispatch_conf'), - brief=('os_builder_linux_ver', - 'cl_builder_disk_size', - 'cl_builder_profile_name'), - next_label=_("Perform"))], + normal=('cl_builder_source_filename', + 'cl_builder_new_id', + 'cl_builder_disk_dev', + 'cl_builder_profile_url', + 'cl_builder_profile_branch', + 'cl_builder_profile_system_shortname', + 'cl_builder_layered_set'), + expert=('cl_templates_locate', + 'cl_verbose_set', + 'cl_dispatch_conf'), + hide=('cl_templates_locate', + 'cl_verbose_set', + 'cl_dispatch_conf'), + brief=('os_builder_linux_ver', + 'os_builder_linux_system', + 'cl_builder_disk_size', + 'cl_builder_profile_data', + 'cl_builder_profile_name'), + next_label=_("Perform"))], 'brief': {'next': __("Perform"), 'name': __("Create the system")}, }, @@ -177,13 +183,24 @@ class Wsdl(WsdlBase): 'native_error': (VariableError, DataVarsError, InstallError, BuilderError, GitError), # значения по умолчанию для переменных этого метода - 'setvars': {'cl_action!': 'image'}, + 'setvars': {'cl_action!': 'image', + 'cl_dispatch_conf': 'usenew'}, # описание груп (список лямбда функций) 'groups': [ - lambda group: group(_("Break the assemble"), - normal=('cl_builder_broken_id',), + lambda group: group(_("Create the image"), + normal=('cl_builder_prepared_id', + 'cl_builder_image_filename'), + brief=('cl_builder_kernel_ver', + 'cl_builder_initrd_install', + 'os_builder_linux_shortname', + 'os_builder_linux_build', + 'os_builder_linux_system', + 'cl_builder_kernel'), expert=('cl_templates_locate', - 'cl_verbose_set', 'cl_dispatch_conf'), - next_label=_("Perform"))] + 'cl_verbose_set', + 'cl_dispatch_conf'), + next_label=_("Perform")),], + 'brief': {'next': __("Perform"), + 'name': __("Create the image")} } ]