Add initrd optimization

pym/cl_fill_install.py

@@ -893,6 +893,14 @@ class fillVars(object, glob_attr):
     def get_os_install_initrd_install(self):
         return self.getInitrd(suffix="install")
 
+    def get_os_install_kernel_config(self):
+        configfile = self.Get('os_install_kernel').replace('vmlinuz', 'config')
+        if configfile.startswith('config') and pathexists(
+                pathjoin(self.Get('cl_chroot_path'),'boot',configfile)):
+            return configfile
+        else:
+            return ""
+
     def get_os_grub_conf(self):
         """Use for generate grub.conf for new system"""
         pathGrubConf = "/boot/grub/grub.conf"
@@ -986,4 +994,4 @@ class fillVars(object, glob_attr):
 
     def get_os_scratch(self):
         """Current system is scratch"""
-        return "on" isMount('/mnt/scratch') else "off"
+        return "on" if isMount('/mnt/scratch') else "off"

pym/cl_install.py

@@ -24,6 +24,7 @@ import traceback
 from os.path import exists as pathexists,join as pathjoin
 from cl_utils import runOsCommand,appendProgramToEnvFile, \
                         removeProgramToEnvFile
+from cl_kernel import KernelConfig,InitRamFs
 
 import cl_overriding
 
@@ -1319,6 +1320,25 @@ class cl_install(color_print):
         #if not self.clVars.WriteVars(header="install"):
         #    raise InstallError(self.getError())
 
+    def cleanInitrd(self):
+        importantSections = ['File systems','Pseudo filesystems',
+                             'CDROM.*Filesystems', 'DOS.*Filesystems',
+                             'SCSI Transport']
+        curConfig = dict(KernelConfig().getSectionParams(*importantSections))
+        newConfig = KernelConfig( configFile=
+            pathjoin(self.clVars.Get('cl_chroot_path'),
+            'boot',self.clVars.Get('os_install_kernel_config')))
+        newConfig = newConfig.getSectionParams(*importantSections)
+        conflictOptions = filter(lambda x: x[1]=="M" and
+                                 x[0] in curConfig and
+                                 curConfig[x[0]]=="Y",
+                                 newConfig)
+        if conflictOptions:
+            return True
+        initrdPath = pathjoin(self.clVars.Get('cl_chroot_path'),
+                                 'boot',self.clVars.Get('os_install_initrd'))
+        return InitRamFs(initrdPath).cleanInitRamFs()
+
     def installSystem(self, force=False, bootDisk=None, stdinReadPwd=False):
         """install System by current variable enviroment"""
         sourceDistr = None
@@ -1475,6 +1495,9 @@ the system") + " (yes/no)"
                 else:
                     self.prepareBoot(targetDistr)
                 self.printByResult(True)
+                # optimize initrd
+                self.printMessageForTest(_("Create new initrd file"))
+                self.printByResult(self.cleanInitrd())
             else:
                 return False
         except (EOFError), e:

pym/cl_kernel.py

@@ -0,0 +1,203 @@
+#-*- coding: utf-8 -*-
+
+# Copyright 2010 Calculate Ltd. http://www.calculate-linux.org
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from cl_utils import typeFile,runOsCommand
+import gzip
+import re
+import os
+from os.path import join as pathjoin
+from os import access,R_OK
+from subprocess import Popen,PIPE
+from shutil import rmtree
+
+class KernelConfig:
+    """Object for work with kernel config
+
+    Example:
+    # get current config
+    kc = KernelConfig()
+    # get specified config
+    kc = KernelConfig(configFile="/boot/config-2.6.34.1-i686-CLD")
+    """
+    def __init__(self,configFile=None):
+        if configFile == None:
+            configFile = '/proc/config.gz'
+        self.config = self._processConfig(configFile)
+
+    def _processConfig(self,configFile):
+        fileType = typeFile().getMType(configFile).partition(';')[0].strip()
+        if fileType == "application/x-gzip":
+            config = gzip.open(configFile)
+        else:
+            config = open(configFile,'r')
+        reEmptyString = re.compile("^(#\s*|\s*)$")
+        # discard empty strings
+        data = filter(lambda x:not reEmptyString.match(x),config.readlines())
+        # convert param names
+        data = map(lambda x:x.strip().replace("=y","=Y"
+                    ).replace("=m","=M"
+                    ).replace(" is not set","=N"), data)
+        # fix data
+        data = ["# Root"]+map(lambda x: x[2:] if "=N" in x else x,data)[3:]
+        
+        config = []
+        curSect = None
+        for line in data:
+            if line.startswith('#'):
+                curSect = line[2:]
+            else:
+                config.append([curSect]+line.split('='))
+        return config
+
+    def getSectionParams(self,*regs):
+        """Get params from sections specified by regular
+
+        Example:
+        params = kernelconfig.getSectionParams("SCSI Transport","File Sys.*")
+        """
+        params = []
+        for reg in regs:
+            reMatch = re.compile(reg,re.M)
+            params += filter(lambda x:reMatch.search(x[0]),self.config)
+        return map(lambda x: (x[1],x[2]), params)
+
+    def getSections(self):
+        return list(set(map(lambda x:x[0],self.config)))
+
+
+class KernelModules:
+    """Object for work with current kernel modules"""
+    def __init__(self):
+        self._initModulesData()
+
+    def _initModulesData(self):
+        def holderlistdir(dirpath):
+            dirs = os.listdir(pathjoin(sysModules,dirpath,"holders"))
+            return reduce(lambda x,y:x+[y]+holderlistdir(pathjoin(dirpath,"holders",y)),
+                          dirs, [])
+        sysModules = '/sys/module'
+        holderPath = lambda x: pathjoin(sysModules,x,"holders")
+        refPath = lambda x: pathjoin(sysModules,x,"refcnt")
+        self.loadModules = filter(lambda x: access(holderPath(x),R_OK) and
+                                       access(refPath(x),R_OK),
+                             os.listdir(sysModules))
+        self.useModules = map(lambda x:int(open(refPath(x),'r').read().strip()),
+                                     self.loadModules)
+        self.depModules = map(lambda x: holderlistdir(x),
+                              self.loadModules)
+
+    def getWorkModules(self,forceRemove=[],forceKeep=[]):
+        """Get work modules"""
+        needModules = filter(lambda x:x[1] > len(x[2]),
+                             zip(self.loadModules,
+                                 self.useModules,
+                                 self.depModules))
+        needModulesName = set(map(lambda x:x[0],needModules)+forceKeep)
+        allModules = map(lambda x:x[0],
+                         filter(lambda x:set(x[1])&needModulesName,
+                                zip(self.loadModules,
+                                    self.depModules)))
+        reRemove = re.compile("|".join(forceRemove))
+        return filter(lambda x: not reRemove.search(x),
+                      map(lambda x:x.replace("_","-"),
+                          allModules+list(needModulesName)))
+class InitRamFs:
+    def __init__(self,initRamFsFile):
+        self.initrdFile = initRamFsFile
+        self.tmpPath = pathjoin('/tmp',os.path.basename(self.initrdFile))
+
+    def _unpackInitRamfs(self):
+        """Unpack initramfs"""
+        self.prevDir = os.getcwd()
+        if not os.path.exists(self.tmpPath):
+            os.mkdir(self.tmpPath)
+        os.chdir(self.tmpPath)
+        cpioProcess = Popen(["cpio", "-di"],
+                            stdout=PIPE,stdin=PIPE,stderr=PIPE)
+        gzipInitrd = gzip.open(self.initrdFile,'r')
+        cpioProcess.stdin.write(gzipInitrd.read())
+        cpioProcess.stdin.flush()
+        cpioProcess.communicate()
+        os.chdir(self.prevDir)
+        return True if cpioProcess.returncode == 0 else False
+
+    def _packInitRamfs(self):
+        """Pack initramfs"""
+        self.prevDir = os.getcwd()
+        try:
+            # get file list for pack
+            fileList = reduce(lambda x,y: x
+                                        +map(lambda z:pathjoin(y[0],z),y[1])
+                                        +map(lambda z:pathjoin(y[0],z),y[2]),
+                              os.walk(self.tmpPath),
+                              [])
+            fileList = map(lambda x:x[len(self.tmpPath)+1:],fileList)
+            # change dir for cpio
+            os.chdir(self.tmpPath)
+            # create cpio process
+            cpioProcess = Popen(["cpio", "-o", "--quiet", "-H","newc"],
+                                stdout=PIPE,stdin=PIPE,stderr=PIPE)
+            # send file list to cpio process
+            cpioProcess.stdin.write("\n".join(fileList))
+            cpioProcess.stdin.write("\n")
+            cpioProcess.stdin.flush()
+            # get result of cpio
+            cpioData = cpioProcess.communicate()[0]
+            cpioRes = cpioProcess.returncode
+        except KeyboardInterrupt:
+            os.chdir(self.prevDir)
+            raise KeyboardInterrupt
+        except Exception,e:
+            print e.__repr__()
+            cpioRes = 255
+        finally:
+            os.chdir(self.prevDir)
+        if cpioRes != 0:
+            return False
+        # remove old initrd file
+        os.unlink(self.initrdFile)
+        # pack and write new initrd file 
+        initrd = gzip.open(self.initrdFile,'w')
+        initrd.write(cpioData)
+        initrd.close()
+        return True
+
+    def cleanInitRamFs(self):
+        """Unpack initramfs clean and pack"""
+        if not self._unpackInitRamfs():
+            return False
+        importantModules = KernelModules().getWorkModules(
+                                forceKeep=['squashfs','zlib_inflate','aufs',
+                                           'exportfs','unionfs','dm_mod'],
+                                forceRemove=['cifs','nfs','snd'])
+        # get walk data with modules files not exists in importantModules
+        deleteModulesData = \
+            map(lambda x:(x[0],
+                          x[1],
+                          filter(lambda x:x.endswith(".ko") and 
+                                          not x[:-3] in importantModules,
+                                 x[2])),
+                os.walk(self.tmpPath))
+        # get full path for all delete modules 
+        deleteModulesData = \
+            reduce(lambda x,y:x+y,
+                   map(lambda x:map(lambda y:pathjoin(x[0],y),x[2]),
+                       deleteModulesData),
+                   [])
+        map(lambda x:os.unlink(x),deleteModulesData)
+        res = self._packInitRamfs()
+        rmtree(self.tmpPath,ignore_errors=True)
+        return res

pym/cl_vars_install.py

@@ -265,6 +265,9 @@ class Data:
     # install initramfs filename
     os_install_initrd_install = {}
 
+    # install config kernel filename
+    os_install_kernel_config = {}
+
     # filesystem format support by calcualte-install
     os_format_type = {}