Added kernel compilation.

Kernel compilation, renaming kernel files, install from installed files.
Initrd optimization.

Need lib with cl_kernel_uid variable. And calculate-install with
ability initramfs optimization in other files.

pym/cl_fill_builder.py

@@ -24,29 +24,56 @@ from os import path
 
 class fillVars(object, glob_attr):
 
-    def get_cl_kernel_directory(self):
+    def get_cl_kernel_src_path(self):
         defaultKernelPath = '/usr/src'
+        priorityDir = path.join(defaultKernelPath,"linux")
         if not access(defaultKernelPath,R_OK):
             return ""
         kernel = cl_kernel()
-        kernelDirs = filter(kernel._testKernelDirectory,os.listdir('/usr/src'))
+        kernelDirs = filter(kernel._testKernelDirectory,
+                            map(lambda x:path.join(defaultKernelPath,x),
+                                os.listdir(defaultKernelPath)))
         if kernelDirs:
-            if "linux" in kernelDirs:
-                return path.join(defaultKernelPath,"linux")
+            if priorityDir in kernelDirs:
+                return priorityDir
             else:
                 return path.join(defaultKernelPath,kernelDirs[0])
         return ""
 
-    def get_cl_kernel_version(self):
-        kernelMakefile = path.join(self.Get('cl_kernel_directory'),'Makefile')
-        reVerPart = re.compile("^(?:VERSION|^PATCHLEVEL|^SUBLEVEL)\s*=",re.I)
+    def get_cl_kernel_full_ver(self):
+        if self.Get('cl_kernel_config') \
+            and os.access(self.Get('cl_kernel_config'),os.R_OK):
+            localVer = self.getValueFromConfig(
+                        self.Get('cl_kernel_config'),
+                                    'CONFIG_LOCALVERSION')
+            if not localVer:
+                localVer=""
+        else:
+            localVer = ""
+        kernelMakefile = path.join(self.Get('cl_kernel_src_path'),'Makefile')
+        if os.access(kernelMakefile,os.R_OK):
+            extraVer = ""
+            for line in open(kernelMakefile,'r'):
+                if "EXTRAVERSION" in line:
+                    extraVer = line.partition('=')[2].strip()
+                    break
+        else:
+            localVer = ""
+        return "%s%s%s"%(self.Get('cl_kernel_ver'), extraVer, localVer)
+
+    def get_cl_kernel_ver(self):
+        kernelMakefile = path.join(self.Get('cl_kernel_src_path'),'Makefile')
+        reVerPart = re.compile(
+            "^(?:VERSION|^PATCHLEVEL|^SUBLEVEL)\s*=",re.I)
         if access(kernelMakefile,R_OK):
             try:
-                return "%(VERSION)s.%(PATCHLEVEL)s.%(SUBLEVEL)s"% \
+                makeVer = \
+                  "%(VERSION)s.%(PATCHLEVEL)s.%(SUBLEVEL)s"% \
                     dict(map(lambda x:(x[0].strip(),x[2].strip()),
                              map(lambda x:x.partition('='),
                                  filter(reVerPart.search,
                                         open(kernelMakefile,'r')))))
+                return makeVer
             except:
                 pass
         return ""
@@ -55,10 +82,26 @@ class fillVars(object, glob_attr):
         kernelDirs = self.Get('cl_kernel_config_path')
         if not access(kernelDirs,R_OK):
             return ""
+        shortVerSearch = re.search("^\d+\.\d+.\d+",self.Get('cl_kernel_ver'),re.I)
+        if shortVerSearch:
+            shortVer = shortVerSearch.group()
+        else:
+            return ""
         configName = "config-%(system)s-%(march)s-%(ver)s" % \
                 {'system':self.Get('os_linux_system'),
                  'march':self.Get('os_arch_machine'),
-                 'ver':self.Get('cl_kernel_version')}
-        if path.exists(configName):
+                 'ver':shortVer}
+        if path.exists(path.join(kernelDirs,configName)):
             return path.join(kernelDirs,configName)
         return ""
+
+    def get_os_builder_makeopts(self):
+        makeconf = '/etc/make.conf'
+        if access(makeconf,R_OK):
+            makeopts = self.getValueFromConfig('/etc/make.conf',"MAKEOPTS")
+            if makeopts:
+                return makeopts
+        return ""
+
+    def get_cl_kernel_boot_path(self):
+        return path.join(self.Get('cl_kernel_install_path'),"boot")

pym/cl_kernel.py

@@ -23,8 +23,10 @@ import sys
 import traceback
 from os import path
 from cl_utils import process
+from subprocess import STDOUT,PIPE
 from cl_print import color_print
 from cl_datavars import DataVars
+from shutil import copy2 as copy_with_perm
 
 from cl_kernel_utils import KernelConfig,InitRamFs
 
@@ -71,3 +73,201 @@ class cl_kernel(color_print):
         self.clVars.importBuilder()
         self.clVars.flIniFile()
 
+    def makeKernel(self,quiet=True,showMenuConfig=False,noClean=False,
+                   lvmOpt=False,dmraidOpt=False,mdadmOpt=False):
+        """Run kernel compilation"""
+        clVars = self.clVars
+        standardParams = [ "--splash=tty1", "--unionfs",
+                                "--all-ramdisk-modules","--disklabel",
+                                "--slowusb", "--no-save-config"]
+        kernelDir = ["--kerneldir=%s"%clVars.Get('cl_kernel_src_path')]
+        kernelDestination = clVars.Get('cl_kernel_install_path')
+        modulePrefix = ["--module-prefix=%s"%kernelDestination]
+        if not path.exists(kernelDestination):
+            self.printERROR("Not found destination directory '%s'"%
+                                kernelDestination)
+            return False
+        logLevel = ["--loglevel=%d"%(1 if quiet else 2)]
+        makeOpts = clVars.Get('os_builder_makeopts')
+        if makeOpts:
+            makeOpts = ["--makeopts=%s"%makeOpts]
+        else:
+            makeOpts = []
+        menuConfig = ["--menuconfig"] if showMenuConfig else []
+        noClean = ["--no-clean"] if noClean else []
+        kernelConfig = ["--kernel-config=%s"%clVars.Get('cl_kernel_config')]
+        bootDir = clVars.Get('cl_kernel_boot_path')
+        if not path.exists(bootDir):
+            os.makedirs(bootDir,mode=0755)
+        bootDir = ["--bootdir=%s"%bootDir]
+        lvmOpt = ["--lvm"] if lvmOpt else []
+        dmraidOpt = ["--dmraid"] if dmraidOpt else []
+        mdadmOpt = ["--mdadm"] if mdadmOpt else []
+        kernelName = ["--kernname=%s"%clVars.Get('os_linux_system')]
+
+        params = ["genkernel"]+standardParams+kernelDir+modulePrefix+\
+                    logLevel+makeOpts+menuConfig+noClean+kernelConfig+\
+                    bootDir+lvmOpt+dmraidOpt+mdadmOpt+["all"]
+
+        try:
+            genkernelProcess = process(*params,stdout=None,stderr=STDOUT,
+                                       stdin=None)
+            return genkernelProcess.success()
+        except KeyboardInterrupt:
+            self.printERROR("Keyboard interrupt")
+            return False
+
+    def prepareBoot(self):
+        """Rename received by genkernel files"""
+        clVars = self.clVars
+        bootDir = clVars.Get('cl_kernel_boot_path')
+        if not os.access(bootDir,os.W_OK):
+            self.printERROR("No permissions to write to '%s'"%
+                bootDir)
+            return False
+        march = clVars.Get('os_arch_machine')
+        if re.match("^i.86$",march):
+            march = "x86"
+
+        baseConfigName = path.join(clVars.Get('cl_kernel_src_path'),
+                                   ".config")
+        if path.exists(baseConfigName):
+            clVars.Set('cl_kernel_config',baseConfigName,True)
+        kernelFullVer = clVars.Get('cl_kernel_full_ver')
+        suffixName = "genkernel-%(march)s-%(fullver)s"%\
+                     {"march":march,
+                      "fullver":kernelFullVer}
+        fullVerWithoutCalculate = kernelFullVer.replace("-calculate","")
+        newSuffixName = "%s-%s-%s-installed"%(fullVerWithoutCalculate,
+                                           clVars.Get('os_arch_machine'),
+                                           clVars.Get('os_linux_shortname'))
+        baseInitrdName = path.join(bootDir,"initramfs-%s"%suffixName)
+        baseKernelName = path.join(bootDir,"kernel-%s"%suffixName)
+        baseSystemMap = path.join(bootDir,"System.map-%s"%suffixName)
+
+        newInitrdName = path.join(bootDir,"initramfs-%s"%newSuffixName)
+        newKernelName = path.join(bootDir,"vmlinuz-%s"%newSuffixName)
+        newSystemMap = path.join(bootDir,"System.map-%s"%newSuffixName)
+        newConfigName = path.join(bootDir,"config-%s"%newSuffixName)
+
+        try:
+            os.rename(baseInitrdName,newInitrdName)
+            os.rename(baseKernelName,newKernelName)
+            os.rename(baseSystemMap,newSystemMap)
+            copy_with_perm(baseConfigName,newConfigName)
+        except OSError,e:
+            self.printERROR(_("Can not rename kernel files: %s")%e.strerror)
+            return False
+
+        return True
+
+    def _installFile(self,source,target,removeSource=True,symlink=False):
+        """Copy, move or create symlink source file to target. Save old."""
+        def getLinkForTarget(target):
+            """Get symlinks from target dirname which point to target"""
+            baseDir = path.dirname(path.normpath(target))
+            linkFiles = filter(path.islink,map(lambda x:path.join(baseDir,x),
+                                               os.listdir(baseDir)))
+            return filter(lambda x:path.join(baseDir,
+                                      os.readlink(x))==target, linkFiles)
+        # raise IOError if source is not exists
+        open(source,'r').close()
+        targetLinkFiles = getLinkForTarget(target)
+        oldtarget = ""
+        if path.lexists(target):
+            oldtarget = "%s.old" % target
+            if path.lexists(oldtarget):
+                oldTargetLinkFiles = getLinkForTarget(oldtarget)
+                map(os.unlink,oldTargetLinkFiles)
+                os.unlink(oldtarget)
+            os.rename(target,oldtarget)
+        if symlink:
+            if path.dirname(source)==path.dirname(target):
+                os.symlink(path.basename(source),target)
+            else:
+                os.symlink(source,target)
+        elif removeSource:
+            os.rename(source,target)
+        else:
+            copy_with_perm(source,target)
+        if oldtarget:
+            map(os.unlink,targetLinkFiles)
+            map(lambda x:os.symlink(path.basename(oldtarget),x),targetLinkFiles)
+
+    def installBootFiles(self):
+        """Copy -install files to without suffix name, and save old copy.
+
+        initramfs, vmlinuz, System.map, config with suffix installed copy
+        withou suffix. Save old files by append suffix .old.
+        Search link files boot directory link to oldfiles and fix symlink.
+        Create link iniramfs-UUID,vmlinuz-UUID,System.map-UUID.
+        Create initramfs install (copy of initramfs)
+        """
+        clVars = self.clVars
+        bootDir = clVars.Get('cl_kernel_boot_path')
+        kernelFullVer = clVars.Get('cl_kernel_full_ver')
+        fullVerWithoutCalculate = kernelFullVer.replace("-calculate","")
+        suffixName = "%s-%s-%s-installed"%(fullVerWithoutCalculate,
+                                           clVars.Get('os_arch_machine'),
+                                           clVars.Get('os_linux_shortname'))
+        newSuffixName = "%s-%s-%s"%(fullVerWithoutCalculate,
+                                           clVars.Get('os_arch_machine'),
+                                           clVars.Get('os_linux_shortname'))
+        initrdName = path.join(bootDir,"initramfs-%s"%suffixName)
+        kernelName = path.join(bootDir,"vmlinuz-%s"%suffixName)
+        systemMap = path.join(bootDir,"System.map-%s"%suffixName)
+        configName = path.join(bootDir,"config-%s"%suffixName)
+        newInitrdName = path.join(bootDir,"initramfs-%s"%newSuffixName)
+        newInitrdNameInstall = path.join(bootDir,
+                                         "initramfs-%s-install"%newSuffixName)
+        newKernelName = path.join(bootDir,"vmlinuz-%s"%newSuffixName)
+        newSystemMap = path.join(bootDir,"System.map-%s"%newSuffixName)
+        newConfigName = path.join(bootDir,"config-%s"%newSuffixName)
+
+        kernelUid = clVars.Get('cl_kernel_uid')
+        symlinkInitrdName = path.join(bootDir,"initramfs-%s"%kernelUid)
+        symlinkInitrdNameInstall = path.join(bootDir,
+                                         "initramfs-%s-install"%kernelUid)
+        symlinkKernelName = path.join(bootDir,"vmlinuz-%s"%kernelUid)
+        symlinkSystemMap = path.join(bootDir,"System.map-%s"%kernelUid)
+        try:
+            self._installFile(initrdName,newInitrdName,removeSource=False)
+            self._installFile(initrdName,newInitrdNameInstall)
+            self._installFile(kernelName,newKernelName)
+            self._installFile(systemMap,newSystemMap)
+            self._installFile(configName,newConfigName)
+
+            self._installFile(newInitrdName,symlinkInitrdName, symlink=True)
+            self._installFile(newInitrdNameInstall,symlinkInitrdNameInstall,
+                              symlink=True)
+            self._installFile(newKernelName,symlinkKernelName, symlink=True)
+            self._installFile(newSystemMap, symlinkSystemMap, symlink=True)
+        except (OSError,IOError),e:
+            self.printERROR(_("Can not install kernel files: %s")%e.strerror)
+            return False
+        return True
+
+    def cleanInitrd(self):
+        clVars = self.clVars
+        bootDir = clVars.Get('cl_kernel_boot_path')
+        clKernelUid = clVars.Get('cl_kernel_uid')
+        initrdName = path.join(bootDir,"initramfs-%s-install"%clKernelUid)
+        optInitrdName = path.join(bootDir,"initramfs-%s"%clKernelUid)
+        # old mode (for compatibility)
+        if not path.lexists(initrdName) or not path.lexists(optInitrdName):
+            kernelFullVer = clVars.Get('cl_kernel_full_ver')
+            fullVerWithoutCalculate = kernelFullVer.replace("-calculate","")
+            suffixName = "%s-%s-%s"%(fullVerWithoutCalculate,
+                                               clVars.Get('os_arch_machine'),
+                                               clVars.Get('os_linux_shortname'))
+            initrdName = path.join(bootDir,"initramfs-%s-install"%suffixName)
+            optInitrdName = path.join(bootDir,"initramfs-%s"%suffixName)
+
+        if path.exists(path.realpath(optInitrdName)):
+            os.unlink(path.realpath(optInitrdName))
+        try:
+            initRamFs = InitRamFs(initrdName)
+            return initRamFs.cleanInitRamFs(path.realpath(optInitrdName))
+        except (OSError,IOError),e:
+            self.printERROR(str(e))
+            return False

pym/cl_kernel_cmd.py

@@ -29,41 +29,44 @@ DESCRIPTION = _("The Calculate Linux kernel builder")
 CMD_OPTIONS = [{'shortOption':"c",
                 'longOption':"kernel-config",
                 'optVal':"FILE",
-                'help':_("Kernel configuration file to use for compilation")
+                'help':_("kernel configuration file to use for compilation")
                },
                {'longOption':"dmraid",
-                'help':_("Include DMRAID support")
+                'help':_("include DMRAID support")
                },
                {'shortOption':"e",
                 'longOption':"extraversion",
                 'optVal':"VER",
-                'help':_("Specify extraversion for kernel")
+                'help':_("specify extraversion for kernel")
                },
                {'shortOption':"k",
                 'longOption':"kerneldir",
                 'optVal':"DIR",
-                'help':_("Location of the kernel sources")
+                'help':_("location of the kernel sources")
                },
                {'longOption':"lvm",
-                'help':_("Include LVM support")
+                'help':_("include LVM support")
                },
                { 'longOption':"mdadm",
-                'help':_("Copy /etc/mdadm.conf to initramfs")
+                'help':_("copy /etc/mdadm.conf to initramfs")
                },
                {'shortOption':"m",
                 'longOption':"menuconfig",
-                'help':_("Run menuconfig after oldconfig")
+                'help':_("run menuconfig after oldconfig")
                },
                {'longOption':"no-clean",
-                'help':_("Do not run make clean before compilation")
+                'help':_("do not run make clean before compilation")
                },
                {'shortOption':"o",
                 'longOption':"use-own-config",
-                'help':_("Use config from kernel directory")
+                'help':_("use config from kernel directory")
                },
                {'shortOption':"q",
-                'help':_("Do not display kernel compilation process")
-               }]
+                'help':_("do not display kernel compilation process")
+               },
+               {'longOption':"initrd",
+                'help':_("perform current initramfs optimization")}
+               ]
 
 
 class kernel_cmd(share_cmd):
@@ -76,6 +79,8 @@ class kernel_cmd(share_cmd):
                           opt.color_control,
                           check_values=self.checkOpts)
         self.logicObj = cl_kernel()
+        self.optionsInitrdIncompatible = ["o","no_clean","m","mdadm","lvm",
+                                          "k", "e","dmraid","c" ]
 
     def _getNamesAllSetOptions(self):
         """Get list set options"""
@@ -90,40 +95,74 @@ class kernel_cmd(share_cmd):
         return ", ".join(map(lambda x: len(x) == 1 and "'-%s'"%x or "'--%s'"%x,
                              listOpt))
 
-    def checkIncompatibleLive(self):
+    def checkIncompatibleInitrd(self):
         """Check incompatible options for option --live"""
         incompatible = list(set(self._getNamesAllSetOptions()) &
-                            set(self.optionsLiveIncompatible))
+                            set(self.optionsInitrdIncompatible))
         if incompatible:
             self.optobj.error(_("incompatible options")+":"+" %s"\
-                    %self.getStringIncompatibleOptions(incompatible+["live"]))
+                    %self.getStringIncompatibleOptions(incompatible+["initrd"]))
 
     def checkOpts(self, values, args):
         """Check values all specified options."""
+        if len(args) > 0:
+            self.optobj.error(_("unrecognized option") + ": %s"% "".join(args))
+        if values.initrd:
+            self.checkIncompatibleInitrd()
         if values.k:
             if not self.logicObj._testKernelDirectory(values.k):
                 self.optobj.error("%s:'%s'"%
-                    (_("Wrong kernel source directory"),values.k))
+                    (_("wrong kernel source directory"),values.k))
             else:
-                self.logic.clVars.Set('os_kernel_directory',values.k,True)
+                self.logicObj.clVars.Set('cl_kernel_src_path',values.k,True)
         if values.c and values.o:
-            self.optobj.error(self.getStringIncompatibleOptions(["c","o"]))
+            self.optobj.error("%s: %s"%(_("incompatible options"),
+                              self.getStringIncompatibleOptions(["c","o"])))
         if values.c:
             if not path.exists(values.c):
-                self.optobj.error(_("Kernel config '%s' not found")%values.c)
+                self.optobj.error(_("kernel config '%s' not found")%values.c)
             else:
-                self.logic.clVars.Set('os_kernel_config',values.c,True)
+                if values.c == path.join(
+                    self.logicObj.clVars.Get('cl_kernel_src_path'),".config"):
+                    configFile = "%s.bak"%values.c
+                else:
+                    configFile = values.c
+                self.logicObj.clVars.Set('cl_kernel_config',configFile,True)
         elif values.o:
             if not path.exists(
-                path.join(self.logic.clVars.Get('os_kernel_directory'),
+                path.join(self.logicObj.clVars.Get('cl_kernel_src_path'),
                           ".config")):
-                    self.optobj.error(_("Kernel directory has not config"))
+                    self.optobj.error(_("kernel directory has not config"))
             else:
-                self.logic.clVars.Set('os_kernel_config',
-                    path.join(self.logic.clVars.Get('os_kernel_directory'),
+                self.logicObj.clVars.Set('cl_kernel_config',
+                    path.join(self.logicObj.clVars.Get('cl_kernel_src_path'),
                           ".config.bak"),True)
 
+        self.optobj.checkVarSyntax(values)
         return (values, args)
 
-    def makeKernel(self):
-        return self.logicObj.makeKernel()
+    def cleanInitrd(self,options):
+        if not self.logicObj.cleanInitrd():
+            self.printERROR(_("Failed to optimize initramfs"))
+            return False
+        else:
+            self.printSUCCESS(_("Initramfs successfully optimized"))
+            return True
+
+    def makeKernel(self,options):
+        if not self.logicObj.makeKernel(quiet=options.q,
+                                        showMenuConfig=options.m,
+                                        noClean=options.no_clean,
+                                        lvmOpt=options.lvm,
+                                        dmraidOpt=options.dmraid,
+                                        mdadmOpt=options.mdadm):
+            self.printERROR(_("Failed kernel compilation"))
+            return False
+        if not self.logicObj.prepareBoot():
+            self.printERROR(_("Failed prepare boot directory"))
+            return False
+        if not self.logicObj.installBootFiles():
+            self.printERROR(_("Failed install kernel"))
+            return False
+        return True
+

pym/cl_vars_builder.py

@@ -31,11 +31,27 @@ class Data:
     # program version
     cl_ver = {'value':__version__}
 
-    cl_kernel_directory = {}
+    # kernel sources path
+    cl_kernel_src_path = {}
 
+    # kernel config path
     cl_kernel_config = {}
 
+    # path which has configs
     cl_kernel_config_path = {'mode':'w',
                     'value':'/var/lib/layman/calculate/profiles/kernel'}
 
-    cl_kernel_version = {}
+    # kernel version
+    cl_kernel_ver = {}
+
+    # kernel full version (added extraversion and localversion)
+    cl_kernel_full_ver = {}
+
+    # make opts received from make.conf
+    os_builder_makeopts = {'mode':'w'}
+
+    # path for vmlinuz,initramfs,system.map and config
+    cl_kernel_boot_path = {'mode':'w'}
+
+    # root path for kernel installation (value+lib/modules)
+    cl_kernel_install_path = {'mode':'w','value':'/'}

scripts/cl-kernel

@@ -50,7 +50,9 @@ if __name__ == "__main__":
     if not kernel.isRoot():
         sys.exit(1)
 
-    if not kernel.makeKernel():
+    if options.initrd:
+        kernel.cleanInitrd(options)
+    elif not kernel.makeKernel(options):
         sys.exit(1)
     
     sys.exit(0)