From de190ad068bb79397c17894bb1f8b3c5af3c716c Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Fri, 24 Sep 2010 18:07:54 +0400 Subject: [PATCH] Start changes --- pym/cl_builder.py | 391 +++++++++++++++++++++++++++++++++++++++++ pym/cl_fill_builder.py | 60 +++++++ pym/cl_vars_builder.py | 50 ++++++ 3 files changed, 501 insertions(+) create mode 100644 pym/cl_builder.py diff --git a/pym/cl_builder.py b/pym/cl_builder.py new file mode 100644 index 0000000..b156a7b --- /dev/null +++ b/pym/cl_builder.py @@ -0,0 +1,391 @@ +#-*- coding: utf-8 -*- + +# Copyright 2010 Calculate Ltd. http://www.calculate-linux.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "2.2.0.0" +__app__ = "calculate-builder" + +import os +import re +import sys +import traceback +from os import path +from cl_utils import process +from subprocess import STDOUT,PIPE +from cl_print import color_print +from cl_datavars import DataVars +from shutil import copy2 as copy_with_perm + +from cl_kernel_utils import KernelConfig,InitRamFs + +from cl_lang import lang + +class printNoColor: + def colorPrint(self,attr,fg,bg,string): + sys.stdout.write(string) + +class DataVarsBuilder(DataVars): + """Variable class for installation""" + + def importBuilder(self, **args): + '''Get variables for builder''' + # section name in calculate.env + envSection = "builder" + # import builder variables + self.importData(envSection, ('cl_vars_builder','cl_fill_builder')) + +class cl_kernel(color_print): + """Primary class for kernel manipulation""" + kernelCurPath = '/usr/src/linux' + + def __init__(self): + self.clVars = None + self.startMessage = "" + + def _testKernelDirectory(self,dirpath): + """Test directory for kernel sources""" + makefilepath = path.join(dirpath,'Makefile') + kbuildpath = path.join(dirpath,'Kbuild') + if not path.exists(makefilepath) \ + or not path.exists(kbuildpath) \ + or not "Kbuild for top-level directory of the kernel" in \ + open(kbuildpath,'r').read(): + return False + return True + + def setNoColor(self): + self.color = False + + def initVars(self): + """Primary initialization of variables""" + self.clVars = DataVarsBuilder() + self.clVars.importBuilder() + self.clVars.flIniFile() + + def makeKernel(self,quiet=True,showMenuConfig=False,noClean=False, + lvmOpt=False,dmraidOpt=False,mdadmOpt=False, + mrproper=False): + """Run kernel compilation""" + clVars = self.clVars + standardParams = ["--splash=tty1", "--unionfs", + "--all-ramdisk-modules","--disklabel", + "--slowusb", "--no-save-config"] + kernelDir = ["--kerneldir=%s"%clVars.Get('cl_kernel_src_path')] + kernelDestination = clVars.Get('cl_kernel_install_path') + modulePrefix = ["--module-prefix=%s"%kernelDestination] + if not path.exists(kernelDestination): + self.printERROR("Not found destination directory '%s'"% + kernelDestination) + return False + logLevel = ["--loglevel=%d"%(1 if quiet else 2)] + makeOpts = clVars.Get('os_builder_makeopts') + if makeOpts: + makeOpts = ["--makeopts=%s"%makeOpts] + else: + makeOpts = [] + menuConfig = ["--menuconfig"] if showMenuConfig else [] + noClean = ["--no-clean"] if noClean else [] + kernelConfig = ["--kernel-config=%s"%clVars.Get('cl_kernel_config')] + bootDir = clVars.Get('cl_kernel_boot_path') + if not path.exists(bootDir): + os.makedirs(bootDir,mode=0755) + bootDir = ["--bootdir=%s"%bootDir] + lvmOpt = ["--lvm"] if lvmOpt else [] + dmraidOpt = ["--dmraid"] if dmraidOpt else [] + mdadmOpt = ["--mdadm"] if mdadmOpt else [] + mrproperOpt = ["--mrproper"] if mrproper else [] + kernelName = ["--kernname=%s"%clVars.Get('os_linux_system')] + cachedir = ["--cachedir=%s"%clVars.Get('cl_kernel_cache_path')] + tempdir = ["--tempdir=%s"%clVars.Get('cl_kernel_temp_path')] + + params = ["genkernel"]+cachedir+tempdir+\ + standardParams+kernelDir+modulePrefix+\ + logLevel+makeOpts+menuConfig+noClean+kernelConfig+\ + bootDir+lvmOpt+dmraidOpt+mdadmOpt+mrproperOpt+["all"] + + try: + genkernelProcess = process(*params,stdout=None,stderr=STDOUT, + stdin=None,envdict=os.environ) + return genkernelProcess.success() + except KeyboardInterrupt: + self.printERROR("Keyboard interrupt") + return False + + def prepareBoot(self): + """Rename received by genkernel files""" + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + if not os.access(bootDir,os.W_OK): + self.printERROR(_("No permissions to write to '%s'")% bootDir) + return False + march = clVars.Get('os_arch_machine') + if re.match("^i.86$",march): + march = "x86" + + baseConfigName = path.join(clVars.Get('cl_kernel_src_path'), + ".config") + if path.exists(baseConfigName): + clVars.Set('cl_kernel_config',baseConfigName,True) + kernelFullVer = clVars.Get('cl_kernel_full_ver') + suffixName = "genkernel-%(march)s-%(fullver)s"%\ + {"march":march, + "fullver":kernelFullVer} + baseInitrdName = path.join(bootDir,"initramfs-%s"%suffixName) + baseKernelName = path.join(bootDir,"kernel-%s"%suffixName) + baseSystemMap = path.join(bootDir,"System.map-%s"%suffixName) + + newInitrdName = self._getName("initramfs") + newKernelName = self._getName("vmlinuz") + newSystemMap = self._getName("System.map") + newConfigName = self._getName("config") + + try: + os.rename(baseInitrdName,newInitrdName) + os.rename(baseKernelName,newKernelName) + os.rename(baseSystemMap,newSystemMap) + copy_with_perm(baseConfigName,newConfigName) + except OSError,e: + self.printERROR(_("Can not rename kernel files: %s")%e.strerror) + return False + return True + + def _installFile(self,source,target,removeSource=True,symlink=False): + """Copy, move or create symlink source file to target. Save old.""" + def getLinkForTarget(target): + """Get symlinks from target dirname which point to target""" + baseDir = path.dirname(path.normpath(target)) + linkFiles = filter(path.islink,map(lambda x:path.join(baseDir,x), + os.listdir(baseDir))) + return filter(lambda x:path.join(baseDir, + os.readlink(x))==target, linkFiles) + # raise IOError if source is not exists + open(source,'r').close() + targetLinkFiles = getLinkForTarget(target) + oldtarget = "" + if path.lexists(target): + oldtarget = "%s.old" % target + if path.lexists(oldtarget): + oldTargetLinkFiles = getLinkForTarget(oldtarget) + map(os.unlink,oldTargetLinkFiles) + os.unlink(oldtarget) + os.rename(target,oldtarget) + if symlink: + if path.dirname(source)==path.dirname(target): + os.symlink(path.basename(source),target) + else: + os.symlink(source,target) + elif removeSource: + os.rename(source,target) + else: + copy_with_perm(source,target) + if oldtarget: + map(os.unlink,targetLinkFiles) + map(lambda x:os.symlink(path.basename(oldtarget),x),targetLinkFiles) + + def _getName(self,obj): + """Get names for (initramfs,initramfs-install,vmlinuz,System.map, + config) after kernel compilation (installed)""" + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + kernelFullVer = clVars.Get('cl_kernel_full_ver') + fullVerWithoutCalculate = kernelFullVer.replace("-calculate","") + suffixName = "%s-%s-%s-installed"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + return path.join(bootDir,{"initramfs":"initramfs-%s", + "initramfs-install":"initramfs-%s", + "vmlinuz":"vmlinuz-%s", + "System.map":"System.map-%s", + "config":"config-%s"}[obj]%suffixName) + + def _getNewName(self,obj): + """Get new names for (initramfs,initramfs-install,vmlinuz,System.map, + config) which they have after renaming""" + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + kernelFullVer = clVars.Get('cl_kernel_full_ver') + fullVerWithoutCalculate = kernelFullVer.replace("-calculate","") + newSuffixName = "%s-%s-%s"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + return path.join(bootDir,{"initramfs":"initramfs-%s", + "initramfs-install":"initramfs-%s-install", + "vmlinuz":"vmlinuz-%s", + "System.map":"System.map-%s", + "config":"config-%s"}[obj]%newSuffixName) + + def _getSymlink(self,obj): + """Get uid symlinks (initramfs,initramfs-install,vmlinuz,System.map) + they pointet to object by _getNewName""" + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + kernelUid = clVars.Get('cl_kernel_uid') + return path.join(bootDir,{"initramfs":"initrd-%s", + "initramfs-install":"initrd-%s-install", + "vmlinuz":"vmlinuz-%s", + "System.map":"System.map-%s"}[obj]%kernelUid) + + def createUidSymlinks(self): + """Create link iniramfs-UUID,vmlinuz-UUID,System.map-UUID.""" + if self.skipByChrootAndShortname(): + return True + newInitrdName = self._getNewName("initramfs") + newInitrdNameInstall = self._getNewName("initramfs-install") + newKernelName = self._getNewName("vmlinuz") + newSystemMap = self._getNewName("System.map") + symlinkInitrdName = self._getSymlink("initramfs") + symlinkInitrdNameInstall = self._getSymlink("initramfs-install") + symlinkKernelName = self._getSymlink("vmlinuz") + symlinkSystemMap = self._getSymlink("System.map") + + try: + self._installFile(newInitrdName,symlinkInitrdName, symlink=True) + self._installFile(newInitrdNameInstall,symlinkInitrdNameInstall, + symlink=True) + self._installFile(newKernelName,symlinkKernelName, symlink=True) + self._installFile(newSystemMap, symlinkSystemMap, symlink=True) + except IOError,e: + self.printERROR( + _("Can not create symlink to current kernel: %s '%s'")% + (e.strerror,e.filename)) + self.printERROR(_("May be kernel was not compiled")) + return False + except OSError,e: + self.printERROR(_("Can not create symlink to current kernel: %s")% + e.strerror) + return False + return True + + + def installBootFiles(self): + """Copy -install files to without suffix name, and save old copy. + + initramfs, vmlinuz, System.map, config with suffix installed copy + withou suffix. Save old files by append suffix .old. + Search link files boot directory link to oldfiles and fix symlink. + Create initramfs install (copy of initramfs) + """ + initrdName = self._getName("initramfs") + kernelName = self._getName("vmlinuz") + systemMap = self._getName("System.map") + configName = self._getName("config") + newInitrdName = self._getNewName("initramfs") + newInitrdNameInstall = self._getNewName("initramfs-install") + newKernelName = self._getNewName("vmlinuz") + newSystemMap = self._getNewName("System.map") + newConfigName = self._getNewName("config") + + try: + self._installFile(initrdName,newInitrdName,removeSource=False) + self._installFile(initrdName,newInitrdNameInstall) + self._installFile(kernelName,newKernelName) + self._installFile(systemMap,newSystemMap) + self._installFile(configName,newConfigName) + except (OSError,IOError),e: + self.printERROR(_("Can not install kernel files: %s")%e.strerror) + return False + return True + + def skipByChrootAndShortname(self): + """Return true if run from chroot or system not Calculate""" + clVars = self.clVars + return clVars.Get('cl_chroot_status') == 'on' or \ + clVars.Get('os_linux_shortname') in ('Gentoo','Linux') + + def versionMigrate(self): + clVars = self.clVars + if self.skipByChrootAndShortname(): + return True + calculate2env = clVars.Get('cl_env_path')[0] + if not path.exists(calculate2env) or \ + not filter(lambda x:x.startswith('cl_kernel_uid'), + open(calculate2env,'r')): + return self.performVersionMigrate() + return True + + def performVersionMigrate(self): + """Generate cl_kernel_uid, write to calculate2.env, fix grub.conf""" + clVars = self.clVars + clKernelUid = clVars.Get('cl_kernel_uid') + clVars.Write('cl_kernel_uid',clKernelUid,force=True) + reChangeKernel = \ + re.compile("(/boot/(?:vmlinuz))(?:-\S+?)?((?:-install)?) ") + reChangeInitrd = \ + re.compile("(/boot/)(?:initrd|initramfs)(?:-\S+?)?((?:-install)?)$") + grubconf = '/boot/grub/grub.conf' + rootdev = clVars.Get('os_root_dev') + def grubreduce(x,y): + if y.startswith('title'): + x[0] = False + elif y.startswith('kernel') and "root=%s"%rootdev in y: + x[0] = True + if x[0]: + y = reChangeKernel.sub("\\1-%s\\2 "%clKernelUid,y) + y = reChangeInitrd.sub("\\1initrd-%s\\2"%clKernelUid,y) + return [x[0],x[1] + [y]] + + if not os.access(grubconf,os.W_OK): + self.printERROR(_("No permissions to write to '%s'")%grubconf) + return False + newGrub = reduce(grubreduce,open(grubconf,'r'),[False,[]])[1] + open(grubconf,'w').writelines(newGrub) + return True + + def setKernelForCurrent(self): + """Set compiled kernel to current""" + clVars = self.clVars + if self.skipByChrootAndShortname(): + return True + # compiling kernel not current. + clKernelPath = clVars.Get('cl_kernel_src_path') + if not clKernelPath in (self.kernelCurPath, + path.realpath(self.kernelCurPath)) and \ + path.islink(self.kernelCurPath): + os.unlink(self.kernelCurPath) + kernelName = path.basename(path.normpath(clKernelPath)) + try: + os.symlink(kernelName,self.kernelCurPath) + except (IOError,OSError),e: + self.printERROR(str(e)) + return False + return True + + def cleanInitrd(self): + """Discard from initrd non-critical modules""" + clVars = self.clVars + if self.skipByChrootAndShortname(): + return True + bootDir = clVars.Get('cl_kernel_boot_path') + clKernelUid = clVars.Get('cl_kernel_uid') + initrdName = path.join(bootDir,"initramfs-%s-install"%clKernelUid) + optInitrdName = path.join(bootDir,"initramfs-%s"%clKernelUid) + # old mode (for compatibility) + if not path.lexists(initrdName) or not path.lexists(optInitrdName): + kernelFullVer = clVars.Get('cl_kernel_full_ver') + fullVerWithoutCalculate = kernelFullVer.replace("-calculate","") + suffixName = "%s-%s-%s"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + initrdName = path.join(bootDir,"initramfs-%s-install"%suffixName) + optInitrdName = path.join(bootDir,"initramfs-%s"%suffixName) + + if path.exists(path.realpath(optInitrdName)): + os.unlink(path.realpath(optInitrdName)) + try: + initRamFs = InitRamFs(initrdName) + return initRamFs.cleanInitRamFs(path.realpath(optInitrdName)) + except (OSError,IOError),e: + self.printERROR(str(e)) + return False diff --git a/pym/cl_fill_builder.py b/pym/cl_fill_builder.py index f7d7ca6..a26b774 100644 --- a/pym/cl_fill_builder.py +++ b/pym/cl_fill_builder.py @@ -144,3 +144,63 @@ class fillVars(object, glob_attr): return "on" if self.isChroot(os.getpid()) else "off" except: return "off" + + def get_cl_builder_iso_action(self): + """Need perform templates for builder:iso""" + pass + + def get_cl_builder_squash_action(self): + """Need perform templates for builder:squahs""" + pass + + def get_cl_builder_squash_path(self): + """Directory of system for livecd.squashfs""" + pass + + def get_cl_builder_iso_path(self): + """Directory for iso building""" + pass + + def get_cl_builder_kernel(self): + """Kernel image""" + pass + + def get_cl_builder_initrd_install(self): + """Initramfs image""" + pass + + def get_os_builder_linux_name(self): + """Build system name""" + pass + + def get_os_builder_linux_subname(self): + """Build system subname""" + pass + + def get_os_builder_linux_shortname(self): + """Build system shortname""" + pass + + def get_os_builder_linux_ver(self): + """Build system ver""" + pass + + def get_os_builder_arch_machine(self): + """Build system arch""" + pass + + def get_os_builder_linux_system(self): + """Build linux system type""" + pass + + def get_cl_builder_cdname(self): + """Cd size specified by name (DVD/CD)""" + pass + + def get_cl_builder_current_squash(self): + """Livecd.squashfs name (may be livecd.squashfs.2 and etc)""" + pass + + def get_cl_builder_image(self): + """Iso image full path""" + pass diff --git a/pym/cl_vars_builder.py b/pym/cl_vars_builder.py index 83798fa..a158016 100644 --- a/pym/cl_vars_builder.py +++ b/pym/cl_vars_builder.py @@ -67,3 +67,53 @@ class Data: # is shell in chroot cl_chroot_status = {} + + # need perform templates for builder:iso + cl_builder_iso_action = {} + + # need perform templates for builder:squahs + cl_builder_squash_action = {} + + # directory of system for livecd.squashfs + cl_builder_squash_path = {} + + # directory for iso building + cl_builder_iso_path = {} + + # kernel image + cl_builder_kernel = {} + + # initramfs image + cl_builder_initrd_install = {} + + # build system name + os_builder_linux_name = {} + + # build system subname + os_builder_linux_subname = {} + + # build system shortname + os_builder_linux_shortname = {} + + # build system ver + os_builder_linux_ver = {} + + # build system arch + os_builder_arch_machine = {} + + # build linux system type + os_builder_linux_system = {} + + # cd size specified by name (DVD/CD) + cl_builder_cdname = {} + + # livecd.squashfs name (may be livecd.squashfs.2 and etc) + cl_builder_current_squash = {} + + # iso image full path + cl_builder_image = {} + + # path which contains images + cl_builder_image_path = {'value':['/var/calculate/remote/linux', + '/var/calculate/linux', + '/usr/calculate/share/linux']}