From 3af5051bce988901d1b6a664f75c48299c1f39b8 Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Tue, 14 Sep 2010 18:02:40 +0000 Subject: [PATCH] Added kernel compilation. Kernel compilation, renaming kernel files, install from installed files. Initrd optimization. Need lib with cl_kernel_uid variable. And calculate-install with ability initramfs optimization in other files. --- pym/cl_fill_builder.py | 63 ++++++++++--- pym/cl_kernel.py | 200 +++++++++++++++++++++++++++++++++++++++++ pym/cl_kernel_cmd.py | 89 ++++++++++++------ pym/cl_vars_builder.py | 20 ++++- scripts/cl-kernel | 4 +- 5 files changed, 338 insertions(+), 38 deletions(-) diff --git a/pym/cl_fill_builder.py b/pym/cl_fill_builder.py index f641d6c..1d4e418 100644 --- a/pym/cl_fill_builder.py +++ b/pym/cl_fill_builder.py @@ -24,29 +24,56 @@ from os import path class fillVars(object, glob_attr): - def get_cl_kernel_directory(self): + def get_cl_kernel_src_path(self): defaultKernelPath = '/usr/src' + priorityDir = path.join(defaultKernelPath,"linux") if not access(defaultKernelPath,R_OK): return "" kernel = cl_kernel() - kernelDirs = filter(kernel._testKernelDirectory,os.listdir('/usr/src')) + kernelDirs = filter(kernel._testKernelDirectory, + map(lambda x:path.join(defaultKernelPath,x), + os.listdir(defaultKernelPath))) if kernelDirs: - if "linux" in kernelDirs: - return path.join(defaultKernelPath,"linux") + if priorityDir in kernelDirs: + return priorityDir else: return path.join(defaultKernelPath,kernelDirs[0]) return "" - def get_cl_kernel_version(self): - kernelMakefile = path.join(self.Get('cl_kernel_directory'),'Makefile') - reVerPart = re.compile("^(?:VERSION|^PATCHLEVEL|^SUBLEVEL)\s*=",re.I) + def get_cl_kernel_full_ver(self): + if self.Get('cl_kernel_config') \ + and os.access(self.Get('cl_kernel_config'),os.R_OK): + localVer = self.getValueFromConfig( + self.Get('cl_kernel_config'), + 'CONFIG_LOCALVERSION') + if not localVer: + localVer="" + else: + localVer = "" + kernelMakefile = path.join(self.Get('cl_kernel_src_path'),'Makefile') + if os.access(kernelMakefile,os.R_OK): + extraVer = "" + for line in open(kernelMakefile,'r'): + if "EXTRAVERSION" in line: + extraVer = line.partition('=')[2].strip() + break + else: + localVer = "" + return "%s%s%s"%(self.Get('cl_kernel_ver'), extraVer, localVer) + + def get_cl_kernel_ver(self): + kernelMakefile = path.join(self.Get('cl_kernel_src_path'),'Makefile') + reVerPart = re.compile( + "^(?:VERSION|^PATCHLEVEL|^SUBLEVEL)\s*=",re.I) if access(kernelMakefile,R_OK): try: - return "%(VERSION)s.%(PATCHLEVEL)s.%(SUBLEVEL)s"% \ + makeVer = \ + "%(VERSION)s.%(PATCHLEVEL)s.%(SUBLEVEL)s"% \ dict(map(lambda x:(x[0].strip(),x[2].strip()), map(lambda x:x.partition('='), filter(reVerPart.search, open(kernelMakefile,'r'))))) + return makeVer except: pass return "" @@ -55,10 +82,26 @@ class fillVars(object, glob_attr): kernelDirs = self.Get('cl_kernel_config_path') if not access(kernelDirs,R_OK): return "" + shortVerSearch = re.search("^\d+\.\d+.\d+",self.Get('cl_kernel_ver'),re.I) + if shortVerSearch: + shortVer = shortVerSearch.group() + else: + return "" configName = "config-%(system)s-%(march)s-%(ver)s" % \ {'system':self.Get('os_linux_system'), 'march':self.Get('os_arch_machine'), - 'ver':self.Get('cl_kernel_version')} - if path.exists(configName): + 'ver':shortVer} + if path.exists(path.join(kernelDirs,configName)): return path.join(kernelDirs,configName) return "" + + def get_os_builder_makeopts(self): + makeconf = '/etc/make.conf' + if access(makeconf,R_OK): + makeopts = self.getValueFromConfig('/etc/make.conf',"MAKEOPTS") + if makeopts: + return makeopts + return "" + + def get_cl_kernel_boot_path(self): + return path.join(self.Get('cl_kernel_install_path'),"boot") diff --git a/pym/cl_kernel.py b/pym/cl_kernel.py index 755d81b..a84eb8b 100644 --- a/pym/cl_kernel.py +++ b/pym/cl_kernel.py @@ -23,8 +23,10 @@ import sys import traceback from os import path from cl_utils import process +from subprocess import STDOUT,PIPE from cl_print import color_print from cl_datavars import DataVars +from shutil import copy2 as copy_with_perm from cl_kernel_utils import KernelConfig,InitRamFs @@ -71,3 +73,201 @@ class cl_kernel(color_print): self.clVars.importBuilder() self.clVars.flIniFile() + def makeKernel(self,quiet=True,showMenuConfig=False,noClean=False, + lvmOpt=False,dmraidOpt=False,mdadmOpt=False): + """Run kernel compilation""" + clVars = self.clVars + standardParams = [ "--splash=tty1", "--unionfs", + "--all-ramdisk-modules","--disklabel", + "--slowusb", "--no-save-config"] + kernelDir = ["--kerneldir=%s"%clVars.Get('cl_kernel_src_path')] + kernelDestination = clVars.Get('cl_kernel_install_path') + modulePrefix = ["--module-prefix=%s"%kernelDestination] + if not path.exists(kernelDestination): + self.printERROR("Not found destination directory '%s'"% + kernelDestination) + return False + logLevel = ["--loglevel=%d"%(1 if quiet else 2)] + makeOpts = clVars.Get('os_builder_makeopts') + if makeOpts: + makeOpts = ["--makeopts=%s"%makeOpts] + else: + makeOpts = [] + menuConfig = ["--menuconfig"] if showMenuConfig else [] + noClean = ["--no-clean"] if noClean else [] + kernelConfig = ["--kernel-config=%s"%clVars.Get('cl_kernel_config')] + bootDir = clVars.Get('cl_kernel_boot_path') + if not path.exists(bootDir): + os.makedirs(bootDir,mode=0755) + bootDir = ["--bootdir=%s"%bootDir] + lvmOpt = ["--lvm"] if lvmOpt else [] + dmraidOpt = ["--dmraid"] if dmraidOpt else [] + mdadmOpt = ["--mdadm"] if mdadmOpt else [] + kernelName = ["--kernname=%s"%clVars.Get('os_linux_system')] + + params = ["genkernel"]+standardParams+kernelDir+modulePrefix+\ + logLevel+makeOpts+menuConfig+noClean+kernelConfig+\ + bootDir+lvmOpt+dmraidOpt+mdadmOpt+["all"] + + try: + genkernelProcess = process(*params,stdout=None,stderr=STDOUT, + stdin=None) + return genkernelProcess.success() + except KeyboardInterrupt: + self.printERROR("Keyboard interrupt") + return False + + def prepareBoot(self): + """Rename received by genkernel files""" + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + if not os.access(bootDir,os.W_OK): + self.printERROR("No permissions to write to '%s'"% + bootDir) + return False + march = clVars.Get('os_arch_machine') + if re.match("^i.86$",march): + march = "x86" + + baseConfigName = path.join(clVars.Get('cl_kernel_src_path'), + ".config") + if path.exists(baseConfigName): + clVars.Set('cl_kernel_config',baseConfigName,True) + kernelFullVer = clVars.Get('cl_kernel_full_ver') + suffixName = "genkernel-%(march)s-%(fullver)s"%\ + {"march":march, + "fullver":kernelFullVer} + fullVerWithoutCalculate = kernelFullVer.replace("-calculate","") + newSuffixName = "%s-%s-%s-installed"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + baseInitrdName = path.join(bootDir,"initramfs-%s"%suffixName) + baseKernelName = path.join(bootDir,"kernel-%s"%suffixName) + baseSystemMap = path.join(bootDir,"System.map-%s"%suffixName) + + newInitrdName = path.join(bootDir,"initramfs-%s"%newSuffixName) + newKernelName = path.join(bootDir,"vmlinuz-%s"%newSuffixName) + newSystemMap = path.join(bootDir,"System.map-%s"%newSuffixName) + newConfigName = path.join(bootDir,"config-%s"%newSuffixName) + + try: + os.rename(baseInitrdName,newInitrdName) + os.rename(baseKernelName,newKernelName) + os.rename(baseSystemMap,newSystemMap) + copy_with_perm(baseConfigName,newConfigName) + except OSError,e: + self.printERROR(_("Can not rename kernel files: %s")%e.strerror) + return False + + return True + + def _installFile(self,source,target,removeSource=True,symlink=False): + """Copy, move or create symlink source file to target. Save old.""" + def getLinkForTarget(target): + """Get symlinks from target dirname which point to target""" + baseDir = path.dirname(path.normpath(target)) + linkFiles = filter(path.islink,map(lambda x:path.join(baseDir,x), + os.listdir(baseDir))) + return filter(lambda x:path.join(baseDir, + os.readlink(x))==target, linkFiles) + # raise IOError if source is not exists + open(source,'r').close() + targetLinkFiles = getLinkForTarget(target) + oldtarget = "" + if path.lexists(target): + oldtarget = "%s.old" % target + if path.lexists(oldtarget): + oldTargetLinkFiles = getLinkForTarget(oldtarget) + map(os.unlink,oldTargetLinkFiles) + os.unlink(oldtarget) + os.rename(target,oldtarget) + if symlink: + if path.dirname(source)==path.dirname(target): + os.symlink(path.basename(source),target) + else: + os.symlink(source,target) + elif removeSource: + os.rename(source,target) + else: + copy_with_perm(source,target) + if oldtarget: + map(os.unlink,targetLinkFiles) + map(lambda x:os.symlink(path.basename(oldtarget),x),targetLinkFiles) + + def installBootFiles(self): + """Copy -install files to without suffix name, and save old copy. + + initramfs, vmlinuz, System.map, config with suffix installed copy + withou suffix. Save old files by append suffix .old. + Search link files boot directory link to oldfiles and fix symlink. + Create link iniramfs-UUID,vmlinuz-UUID,System.map-UUID. + Create initramfs install (copy of initramfs) + """ + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + kernelFullVer = clVars.Get('cl_kernel_full_ver') + fullVerWithoutCalculate = kernelFullVer.replace("-calculate","") + suffixName = "%s-%s-%s-installed"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + newSuffixName = "%s-%s-%s"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + initrdName = path.join(bootDir,"initramfs-%s"%suffixName) + kernelName = path.join(bootDir,"vmlinuz-%s"%suffixName) + systemMap = path.join(bootDir,"System.map-%s"%suffixName) + configName = path.join(bootDir,"config-%s"%suffixName) + newInitrdName = path.join(bootDir,"initramfs-%s"%newSuffixName) + newInitrdNameInstall = path.join(bootDir, + "initramfs-%s-install"%newSuffixName) + newKernelName = path.join(bootDir,"vmlinuz-%s"%newSuffixName) + newSystemMap = path.join(bootDir,"System.map-%s"%newSuffixName) + newConfigName = path.join(bootDir,"config-%s"%newSuffixName) + + kernelUid = clVars.Get('cl_kernel_uid') + symlinkInitrdName = path.join(bootDir,"initramfs-%s"%kernelUid) + symlinkInitrdNameInstall = path.join(bootDir, + "initramfs-%s-install"%kernelUid) + symlinkKernelName = path.join(bootDir,"vmlinuz-%s"%kernelUid) + symlinkSystemMap = path.join(bootDir,"System.map-%s"%kernelUid) + try: + self._installFile(initrdName,newInitrdName,removeSource=False) + self._installFile(initrdName,newInitrdNameInstall) + self._installFile(kernelName,newKernelName) + self._installFile(systemMap,newSystemMap) + self._installFile(configName,newConfigName) + + self._installFile(newInitrdName,symlinkInitrdName, symlink=True) + self._installFile(newInitrdNameInstall,symlinkInitrdNameInstall, + symlink=True) + self._installFile(newKernelName,symlinkKernelName, symlink=True) + self._installFile(newSystemMap, symlinkSystemMap, symlink=True) + except (OSError,IOError),e: + self.printERROR(_("Can not install kernel files: %s")%e.strerror) + return False + return True + + def cleanInitrd(self): + clVars = self.clVars + bootDir = clVars.Get('cl_kernel_boot_path') + clKernelUid = clVars.Get('cl_kernel_uid') + initrdName = path.join(bootDir,"initramfs-%s-install"%clKernelUid) + optInitrdName = path.join(bootDir,"initramfs-%s"%clKernelUid) + # old mode (for compatibility) + if not path.lexists(initrdName) or not path.lexists(optInitrdName): + kernelFullVer = clVars.Get('cl_kernel_full_ver') + fullVerWithoutCalculate = kernelFullVer.replace("-calculate","") + suffixName = "%s-%s-%s"%(fullVerWithoutCalculate, + clVars.Get('os_arch_machine'), + clVars.Get('os_linux_shortname')) + initrdName = path.join(bootDir,"initramfs-%s-install"%suffixName) + optInitrdName = path.join(bootDir,"initramfs-%s"%suffixName) + + if path.exists(path.realpath(optInitrdName)): + os.unlink(path.realpath(optInitrdName)) + try: + initRamFs = InitRamFs(initrdName) + return initRamFs.cleanInitRamFs(path.realpath(optInitrdName)) + except (OSError,IOError),e: + self.printERROR(str(e)) + return False diff --git a/pym/cl_kernel_cmd.py b/pym/cl_kernel_cmd.py index 0896be6..5b67eee 100644 --- a/pym/cl_kernel_cmd.py +++ b/pym/cl_kernel_cmd.py @@ -29,41 +29,44 @@ DESCRIPTION = _("The Calculate Linux kernel builder") CMD_OPTIONS = [{'shortOption':"c", 'longOption':"kernel-config", 'optVal':"FILE", - 'help':_("Kernel configuration file to use for compilation") + 'help':_("kernel configuration file to use for compilation") }, {'longOption':"dmraid", - 'help':_("Include DMRAID support") + 'help':_("include DMRAID support") }, {'shortOption':"e", 'longOption':"extraversion", 'optVal':"VER", - 'help':_("Specify extraversion for kernel") + 'help':_("specify extraversion for kernel") }, {'shortOption':"k", 'longOption':"kerneldir", 'optVal':"DIR", - 'help':_("Location of the kernel sources") + 'help':_("location of the kernel sources") }, {'longOption':"lvm", - 'help':_("Include LVM support") + 'help':_("include LVM support") }, { 'longOption':"mdadm", - 'help':_("Copy /etc/mdadm.conf to initramfs") + 'help':_("copy /etc/mdadm.conf to initramfs") }, {'shortOption':"m", 'longOption':"menuconfig", - 'help':_("Run menuconfig after oldconfig") + 'help':_("run menuconfig after oldconfig") }, {'longOption':"no-clean", - 'help':_("Do not run make clean before compilation") + 'help':_("do not run make clean before compilation") }, {'shortOption':"o", 'longOption':"use-own-config", - 'help':_("Use config from kernel directory") + 'help':_("use config from kernel directory") }, {'shortOption':"q", - 'help':_("Do not display kernel compilation process") - }] + 'help':_("do not display kernel compilation process") + }, + {'longOption':"initrd", + 'help':_("perform current initramfs optimization")} + ] class kernel_cmd(share_cmd): @@ -76,6 +79,8 @@ class kernel_cmd(share_cmd): opt.color_control, check_values=self.checkOpts) self.logicObj = cl_kernel() + self.optionsInitrdIncompatible = ["o","no_clean","m","mdadm","lvm", + "k", "e","dmraid","c" ] def _getNamesAllSetOptions(self): """Get list set options""" @@ -90,40 +95,74 @@ class kernel_cmd(share_cmd): return ", ".join(map(lambda x: len(x) == 1 and "'-%s'"%x or "'--%s'"%x, listOpt)) - def checkIncompatibleLive(self): + def checkIncompatibleInitrd(self): """Check incompatible options for option --live""" incompatible = list(set(self._getNamesAllSetOptions()) & - set(self.optionsLiveIncompatible)) + set(self.optionsInitrdIncompatible)) if incompatible: self.optobj.error(_("incompatible options")+":"+" %s"\ - %self.getStringIncompatibleOptions(incompatible+["live"])) + %self.getStringIncompatibleOptions(incompatible+["initrd"])) def checkOpts(self, values, args): """Check values all specified options.""" + if len(args) > 0: + self.optobj.error(_("unrecognized option") + ": %s"% "".join(args)) + if values.initrd: + self.checkIncompatibleInitrd() if values.k: if not self.logicObj._testKernelDirectory(values.k): self.optobj.error("%s:'%s'"% - (_("Wrong kernel source directory"),values.k)) + (_("wrong kernel source directory"),values.k)) else: - self.logic.clVars.Set('os_kernel_directory',values.k,True) + self.logicObj.clVars.Set('cl_kernel_src_path',values.k,True) if values.c and values.o: - self.optobj.error(self.getStringIncompatibleOptions(["c","o"])) + self.optobj.error("%s: %s"%(_("incompatible options"), + self.getStringIncompatibleOptions(["c","o"]))) if values.c: if not path.exists(values.c): - self.optobj.error(_("Kernel config '%s' not found")%values.c) + self.optobj.error(_("kernel config '%s' not found")%values.c) else: - self.logic.clVars.Set('os_kernel_config',values.c,True) + if values.c == path.join( + self.logicObj.clVars.Get('cl_kernel_src_path'),".config"): + configFile = "%s.bak"%values.c + else: + configFile = values.c + self.logicObj.clVars.Set('cl_kernel_config',configFile,True) elif values.o: if not path.exists( - path.join(self.logic.clVars.Get('os_kernel_directory'), + path.join(self.logicObj.clVars.Get('cl_kernel_src_path'), ".config")): - self.optobj.error(_("Kernel directory has not config")) + self.optobj.error(_("kernel directory has not config")) else: - self.logic.clVars.Set('os_kernel_config', - path.join(self.logic.clVars.Get('os_kernel_directory'), + self.logicObj.clVars.Set('cl_kernel_config', + path.join(self.logicObj.clVars.Get('cl_kernel_src_path'), ".config.bak"),True) + self.optobj.checkVarSyntax(values) return (values, args) - def makeKernel(self): - return self.logicObj.makeKernel() + def cleanInitrd(self,options): + if not self.logicObj.cleanInitrd(): + self.printERROR(_("Failed to optimize initramfs")) + return False + else: + self.printSUCCESS(_("Initramfs successfully optimized")) + return True + + def makeKernel(self,options): + if not self.logicObj.makeKernel(quiet=options.q, + showMenuConfig=options.m, + noClean=options.no_clean, + lvmOpt=options.lvm, + dmraidOpt=options.dmraid, + mdadmOpt=options.mdadm): + self.printERROR(_("Failed kernel compilation")) + return False + if not self.logicObj.prepareBoot(): + self.printERROR(_("Failed prepare boot directory")) + return False + if not self.logicObj.installBootFiles(): + self.printERROR(_("Failed install kernel")) + return False + return True + diff --git a/pym/cl_vars_builder.py b/pym/cl_vars_builder.py index 99cf77b..9876e9e 100644 --- a/pym/cl_vars_builder.py +++ b/pym/cl_vars_builder.py @@ -31,11 +31,27 @@ class Data: # program version cl_ver = {'value':__version__} - cl_kernel_directory = {} + # kernel sources path + cl_kernel_src_path = {} + # kernel config path cl_kernel_config = {} + # path which has configs cl_kernel_config_path = {'mode':'w', 'value':'/var/lib/layman/calculate/profiles/kernel'} - cl_kernel_version = {} + # kernel version + cl_kernel_ver = {} + + # kernel full version (added extraversion and localversion) + cl_kernel_full_ver = {} + + # make opts received from make.conf + os_builder_makeopts = {'mode':'w'} + + # path for vmlinuz,initramfs,system.map and config + cl_kernel_boot_path = {'mode':'w'} + + # root path for kernel installation (value+lib/modules) + cl_kernel_install_path = {'mode':'w','value':'/'} diff --git a/scripts/cl-kernel b/scripts/cl-kernel index 427edbc..e8da802 100644 --- a/scripts/cl-kernel +++ b/scripts/cl-kernel @@ -50,7 +50,9 @@ if __name__ == "__main__": if not kernel.isRoot(): sys.exit(1) - if not kernel.makeKernel(): + if options.initrd: + kernel.cleanInitrd(options) + elif not kernel.makeKernel(options): sys.exit(1) sys.exit(0)