#-*- coding: utf-8 -*- # Copyright 2010 Calculate Ltd. http://www.calculate-linux.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from os import path from random import choice import string import os import types from time import sleep import re import sys import operator from cl_utils import runOsCommand,isMount,removeDir,typeFile,pathJoin, \ process,getRunCommands,getTupleVersion,cmpVersion, \ detectDeviceForPartition, getProgPath,listDirectory, \ checkUtils,STDOUT,getUdevDeviceInfo, countPartitions from cl_vars_share import varsShare from shutil import copyfile,copytree from cl_template import _terms from subprocess import Popen,PIPE,STDOUT from cl_fill import fillVars from cl_vars_share import varsShare import threading from cl_lang import lang tr = lang() tr.setLocalDomain('cl_lib') tr.setLanguage(sys.modules[__name__]) class SignalInterrupt: __interruptProcessObjs = [] __sigint = False def setSignalInterrupt(self): """Handler "SIGINT""" if SignalInterrupt.__sigint is False: sys.stdout.write("\b\b") # killed processes while (SignalInterrupt.__interruptProcessObjs): process = SignalInterrupt.__interruptProcessObjs.pop() if hasattr(process, "pipe") and hasattr(process.pipe,"kill"): process.pipe.kill() SignalInterrupt.__sigint = True def getSignalInterrupt(self): return SignalInterrupt.__sigint def addInterruptProcess(self, process): SignalInterrupt.__interruptProcessObjs.append(process) class Spinner(threading.Thread): stopSignal = threading.Event() sequence= map(lambda x:x,iter("/-\\|")) write = sys.stdout.write selfthread = None def setWriteFunc(self,writeFunc): Spinner.write = writeFunc def init(self,checkFunction=None,interval=0.1): self.curpos = 0 self.checkFunction = checkFunction self.interval = interval Spinner.stopSignal.clear() def stop(self): Spinner.stopSignal.set() if Spinner.selfthread: Spinner.selfthread.join(2) Spinner.selfthread = None #Spinner().write('\n') Spinner().write('\b') def run(self): self.write(self.sequence[-1]) Spinner.selfthread = self while not Spinner.stopSignal.isSet(): if self.checkFunction and self.checkFunction(): self.write("\b%s"%self.sequence[self.curpos]) self.curpos += 1 if self.curpos >= len(self.sequence): self.curpos = 0 sleep(self.interval) class DistributiveError(Exception): """Error for distributive operations""" pass class DistributiveRepository: varsShare = varsShare() contentCache = {} marches = ['i686','x86_64'] extensiton = ['iso','tar.bz2','tar.gz','tar.7z','tar.lzma'] reDistName = re.compile(""" ^.*/(?P%(name)s) -(?P%(ver)s) -(?P%(march)s) .(?P%(ext)s)$""" % {'name':"[a-z0-9]+", 'ver':r"(\d+\.)*\d+", 'march':"|".join(marches), 'ext':"|".join(extensiton) }, re.X) def __init__(self,directories=[]): self.dirs = directories def ini_to_dict(self,filename): """Convert ini values to dict""" if os.access(filename,os.R_OK): return dict(map(lambda x: x.strip().rpartition('=')[0::2], open(filename,'r'))) else: return {} def _getfromcontent(self,filename,addFunc=None): """Get info from content""" origfilename = filename if filename in DistributiveRepository.contentCache: return DistributiveRepository.contentCache[filename].copy() varsShare = self.varsShare distr = None # may be directory is isodir (directory which contains iso image) extname = "isodir" try: distr = IsoDistributive(filename) if not distr.probe() and filename.startswith('/dev'): distr.close() distr = None distr = PartitionDistributive(filename) filename = distr.convertToDirectory().directory except Exception,e: extname = "dir" d = self.ini_to_dict(path.join(filename, 'etc/calculate/calculate.ini')) if not d or not "march" in d: if path.exists(path.join(filename,'lib64')): d['march'] = 'x86_64' else: d['march']= 'i686' if d: d['ext'] = extname d["name"] = varsShare.getShortnameByMakeprofile(filename) or \ varsShare.getShortnameByIni(filename) or \ varsShare.detectOtherShortname(filename) or \ "Linux" d['ver'] = \ varsShare.getVersionFromMetapackage(filename,d["name"]) or \ varsShare.getVersionFromCalculateIni(filename) or "0" reOsLinuxBuild = re.compile("^os_linux_build\s*=\s*(\S+)\s*$") os_linux_build = \ map(lambda x:x.groups()[0], filter(lambda x:x, map(reOsLinuxBuild.search, reduce(lambda x,y:x+y, map(lambda x:open(x,"r").readlines(), filter(path.exists, [path.join(filename,"etc/calculate/calculate2.env")])),[])))) if os_linux_build: d['build'] = os_linux_build[-1] else: d['build'] = "" if addFunc: d = addFunc(filename,d) if distr: distr.close() DistributiveRepository.contentCache[origfilename] = d return d.copy() def _getdistrinfo(self,filename): """Get information by distributive""" varsShare = self.varsShare # if filename is directory if not path.isfile(filename): return self._getfromcontent(filename) else: match = self.reDistName.match(filename) if not match: return {} distdic = match.groupdict() distdic["build"] = "" if "ver" in distdic: if re.match("^\d{8}$", distdic["ver"]): distdic["build"] = distdic["ver"] distdic["ver"] = "" return distdic def _getAvailableShortnames(self): """Get available distributives shortnames""" distros = filter(lambda x:x, map(self.reDistName.search,self._getAvailableDistributives())) return sorted(list(set(map(lambda x:x.groupdict()['name'],distros)))) def _getAvailableDistributives(self,system=None,shortname=None,march=None, version=None,op_compare=None): """Get all distributives by filter""" if op_compare is None: op_compare = operator.eq if version: version = getTupleVersion(version) def distfilter(dist): d = self._getdistrinfo(dist) if not d: return False # check filter conditions if system and self.system(d['name']) != system: return False if not "name" in d or not "ver" in d: return False if shortname and d['name'].lower() != shortname.lower(): return False if march and d['march'] != march: return False if version and not op_compare(getTupleVersion(d['ver']), version): return False return True def listdistr(pathname): if path.exists(path.join(pathname,'etc/make.profile')): return [pathname] elif path.exists(path.join(pathname,'livecd')): return [pathname] elif pathname.startswith('/dev/'): return [pathname] else: # discard inner directories return filter(lambda x:not path.isdir( path.join(pathname,x)), listDirectory(pathname)) # get lists files in directories allFiles = map(lambda x: map(lambda y: path.join(x,y), listdistr(x)), # discard not exists distrib directories filter(lambda x: os.access(x,os.R_OK), self.dirs)) # filter distributives return filter(distfilter, # join files lists to one list reduce(lambda x,y: x + y, allFiles, [])) def getDistributiveByFile(self,filename): """Get Distributive object by filename""" # MAGIC_COMPRESS 0x000004 Check inside compressed files tf = typeFile(magic=0x6|0x20) ftype = tf.getMType(filename) if "block special" in ftype: return IsoDistributive(filename) if "ISO 9660 CD-ROM" in ftype: return IsoDistributive(filename) elif "7-zip" in ftype or \ "POSIX tar archive" in ftype: return ArchiveDistributive(filename) elif "Squashfs filesystem" in ftype: return SquashDistributive(filename) elif path.isdir(filename): if path.exists(path.join(filename,"livecd")): return IsoDistributive(filename) else: return DirectoryDistributive(filename) else: raise DistributiveError(_("Wrong distribution") + " '%s':\n%s"\ %(filename,ftype)) def extcomparator(self,*exts): avexts = ("iso","isodir","dir") vals = [] for i in exts: if i in avexts: vals.append(avexts.index(i)) else: vals.append(-1) return cmp(vals[0],vals[1]) term = _terms() def sortdistrfunc(self,x,y): """Func of comparing two distributive""" ver1, ver2 = x[1].get('ver',""), y[1].get('ver',"") if ver1 and ver2 and ver1 != "0" and ver2 != "0" and ver1 != ver2: return cmpVersion(ver1,ver2) build1,build2 = self.term._convertVers(x[1].get('build',""), y[1].get('build',"")) if build1 != build2: return cmp(build1,build2) else: ext1 = x[1].get('ext',"") ext2 = y[1].get('ext',"") return self.extcomparator(ext1,ext2) def getBestDistributive(self,system=None,shortname=None,march=None, version=None,discardType=[], op_compare=None): """Get the actualest distributive""" availDistrs = self._getAvailableDistributives(system,shortname, march,version, op_compare) availDistrs = filter(lambda x:x[1] and "ext" in x[1] and not x[1]["ext"] in discardType, map(lambda x:(x,self._getdistrinfo(x)), availDistrs)) availDistrs = sorted(availDistrs,self.sortdistrfunc,reverse=True) if availDistrs: return availDistrs[0][0] else: return None def _findLatestFile(self,dirs,reMatch,keyfunc): """Find latest file in dirs, which match by reMatch, comparable part get by keyfunc""" existsdirs = filter(path.exists,dirs) listimgs = reduce(lambda x,y:x + \ map(lambda x:reMatch.search(path.join(y,x)), listDirectory(y)), existsdirs,[]) listimgs = filter(lambda x:x, listimgs) if listimgs: return max(listimgs,key=keyfunc).group() return "" def getBestStage(self,march=None,hardened=None): """Get latest stage by march""" if march: march = {'x86_64':'amd64'}.get(march,march) else: march = "[^-]+" if hardened is None: hardened = "(?:-hardened)?" elif hardened == True: hardened = "-hardened" elif hardened == False: hardened = "" reStage = re.compile(r'^.*/stage3-%s%s-(\d+)\.tar\.bz2$'% (march,hardened),re.S) return self._findLatestFile(self.dirs,reStage,lambda x:x.groups()[0]) class Distributive(object, SignalInterrupt): """Distributive object. Parent object for all distributive.""" mountError = _("Failed to mount") + " %s:\n%s" reLive = re.compile(r"^live[^.]*\.squashfs(\.(\d+))?$",re.S) flagSpinner=True def __init__(self, parent=None): self.childs = [] # if specified parent if parent: # save parent type for correct resource release self.parent = parent # append this object as child to specified parent parent.childs.append(self) else: self.parent = None def detach(self): """Detach child distributive from parent. At example: ArchiveDistributive create child DirectoryDistributive by unpacking to temporary directory and at close method remove it. If the removing directroy do not need, then need call detach in DirectoryDistributive object dist1 = ArchiveDistributive(file="/os.tar.bz2",mdirectory="/tmp/test") dist2 = dist1.convertToDirectory() dist2.detach() dist1.close() ... """ self.parent = None def close(self): """Release all child distributive and release himself. Need call this method at end work with object for right releasing resources. Example: dist1 = PartitionDistributive(partition="/dev/sda2") dist2 = dist1.convertToDirectory() dist1.close() """ # close all child if self.childs: for child in reversed(self.childs): # check detach if child.parent: child.close() self.childs = [] # if has parent if self.parent: self.parent.releaseChild(self) self.parent = None def releaseChild(self,child): """Method of release child state of distributive At example: PartitionDistributive may be DirectoryDistributive by mounting it to directory. But at end this directory must be unmounted.""" pass def convertToDirectory(self): """Default c raise error about impossible convert object""" raise DistributiveError(_("Failed to convert") + " '%s' "\ %self.__class__.__name__ + _("to") +\ " '%s'" %"DirectoryDistributive") # def __del__(self): # """Uncomment this method for automaticaly release all distributive # instance""" # self.close() def runOsCommand(self, *argv, **kwarg): res, mes = runOsCommand(*argv,**kwarg) mes = "\n".join(map(lambda x: x.strip(), mes)) return res,mes def getDirectory(self): """Get directory which contains distro""" return self.convertToDirectory().directory def getBootDirectory(self): """Get directory which contains boot""" return path.join(self.getDirectory(),"boot") def _makeDirectory(self,pathname): """Make directory and parent. If directory exists then return False else True""" try: parent = path.split(path.normpath(pathname))[0] if not path.exists(parent): self._makeDirectory(parent) else: if path.exists(pathname): return False os.mkdir(pathname) return True except Exception, e: raise DistributiveError(_("Failed to create the directory") + " '%s':\n%s" %(pathname,str(e))) except KeyboardInterrupt, e: self.setSignalInterrupt() raise DistributiveError(_("Failed to create the directory") + " '%s':\n%s" %(pathname,str(e))) def _removeDirectory(self,directory): """Remove directory and files contained in it""" try: removeDir(directory) except Exception, e: raise DistributiveError(_("Failed to remove the directory from") +\ " '%s':\n%s" %(directory,str(e))) except KeyboardInterrupt, e: self.setSignalInterrupt() raise DistributiveError(_("Failed to remove the directory from") +\ " '%s':\n%s" %(directory,str(e))) def _copyfile(self,infile,outfile): try: copyfile(infile,outfile) except (Exception),e: raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s"\ %(infile,outfile,str(e))) def _copytree(self,indir,outdir): try: copytree(indir,outdir,symlinks=True) except Exception ,e: raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s"\ %(indir,outdir,str(e))) except KeyboardInterrupt, e: self.setSignalInterrupt() raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s"\ %(indir,outdir,str(e))) def rsync(self,fromdir,todir,hideSpin=False): """Copy files from 'fromdir' directory to 'todir' directory""" if self.flagSpinner and not hideSpin: def checkRsync(): try: return len(filter(lambda x:"rsync\x00-a\x00-H\x00-x" in x, getRunCommands()))>2 except: return False spin = Spinner() spin.init(checkFunction=checkRsync,interval=0.6) spin.start() rsyncCmd = varsShare().getProgPath('/usr/bin/rsync') if not rsyncCmd: raise DistributiveError(_("Failed to find '%s' command")%"rsync") try: rsyncProcess = process(rsyncCmd, "-a","-H", "-x", "%s/"%fromdir,"%s/"%todir,stderr=STDOUT) self.addInterruptProcess(rsyncProcess) res = rsyncProcess.success() errmes = rsyncProcess.read() except Exception,e: res = False errmes = str(e) except KeyboardInterrupt: self.setSignalInterrupt() if self.flagSpinner: spin.stop() raise KeyboardInterrupt() if self.flagSpinner and not hideSpin: spin.stop() if res is True: return True else: if "No space left on device" in errmes: errmes = "No space left on device" raise DistributiveError(_("Failed to copy files from") +\ " '%s' " %fromdir + _("to") +\ " '%s':\n%s" %(todir,errmes)) def _mountToBind(self,srcDirectory,destDirectory): """Mount srcDirectory to destDirectory""" mount = process('/bin/mount',"-o","bind",srcDirectory,destDirectory, stderr=STDOUT) if mount.success(): return True else: try: os.rmdir(destDirectory) except: pass raise DistributiveError(self.mountError%(srcDirectory,mount.read())) def performFormat(self): pass def formatPartition(self,format=""): pass def rndString(self): """Get random string with len 8 char""" return "".join([choice(string.ascii_letters+string.digits) for i in xrange(0,8)]) def _getSquashNum(self,reMatch): if reMatch.groups()[1] and reMatch.groups()[1].isdigit(): return int(reMatch.groups()[1]) else: return 0 def _getLastLive(self,directory): """Get last live squashfs image from directory""" squashfiles = filter(lambda x:x, map(self.reLive.search, listDirectory(directory))) if squashfiles: return max(squashfiles, key=self._getSquashNum).group() return None def _mountToDirectory(self,file,directory,mountopts="",count=2): """Mount squashfs to directory""" NO_SUCH_DEVICE = 2816 if isMount(directory): raise DistributiveError(_("Failed to mount to the directory: %s\n")\ %directory+ _("Directory already mounted")) mountopts = filter(lambda x:x, mountopts.split(" ")) mountProcess = process('/bin/mount',file,directory,*mountopts, stderr=STDOUT) if mountProcess.success(): return True else: #2816 code return by mount if device is absent (update /dev by udev) # try mount 3 times with interval 0.5 second if mountProcess.returncode() == NO_SUCH_DEVICE and count: sleep(0.5) mountProcess.close() return self._mountToDirectory(file,directory,mountopts,count-1) try: self._removeDirectory(directory) except: pass raise DistributiveError(self.mountError%(file,mountProcess.read())) def _umountDirectory(self,directory): """Umount directory""" if isMount(directory): for wait in [0,0.5,2,5]: sleep(wait) processUmount = process('/bin/umount',directory,stderr=STDOUT) if processUmount.success(): return True raise DistributiveError(_("Failed to umount") + " %s:\n%s"% (directory,processUmount.read())) else: return True def _getMntDirectory(self,directory): """Get directory name, which will use for mounting or unpacking If queried name is not free then to name append random string """ newDirectoryName = directory while path.exists(newDirectoryName): newDirectoryName = "%s.%s"%(directory,self.rndString()) return newDirectoryName class DirectoryDistributive(Distributive): def __init__(self,directory,parent=None,mdirectory=None): Distributive.__init__(self,parent=parent) self.directory = directory self.mdirectory = mdirectory if not parent: self._makeDirectory(self.directory) def bindDirectory(self,mdirectory): for child in self.childs: if isinstance(child,DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._makeDirectory(mdirectory) self._mountToBind(self.directory,mdirectory) return DirectoryDistributive(mdirectory,parent=self) def releaseChild(self,child): """Remove child Directory distributive""" if isinstance(child,DirectoryDistributive): self._umountDirectory(child.directory) self._removeDirectory(child.directory) child.directory = None def convertToDirectory(self): if self.mdirectory: return self.bindDirectory(self.mdirectory) else: return self def performFormat(self): """Format for directory - removing all files""" execStr = '/bin/rm -rf --one-file-system %s'%self.directory res,errmes = self.runOsCommand(execStr) if res == 0: return True else: raise DistributiveError(_("Failed to format the partition") + " %s:\n%s"%(dev,errmes)) self._makeDirectory(self.directory) def installFrom(self, source): """Install distributive to directory from source distributive""" if isinstance(source,ArchiveDistributive): source.unpackTo(self.directory) else: # get source distributive as directory distributive dFrom = source.convertToDirectory() # copy distributive from source to this self.rsync(dFrom.directory,self.directory) class DataPartition: """Data partition""" dev = None mountPoint = None fileSystem = "reiserfs" isFormat = False systemId = None partitionTable = None class MultiPartitions: """Data partition list""" listPartitions = [] def addPartition(self, **argv): """Add partition in data partition list""" dictDataPart = reduce(lambda x,y:\ x.update({y:getattr(DataPartition,y)}) or x, filter(lambda x: not x.startswith('_'), DataPartition.__dict__),{}) updateAttrData = filter(lambda x: x[1]!=None, dictDataPart.items()) defaultAttr = [] for attrName, attrValue in updateAttrData: if not attrName in argv.keys(): defaultAttr.append(attrName) argv[attrName] = attrValue if set(argv.keys()) != set(dictDataPart.keys()): notFoundAttr = set(dictDataPart.keys()) - set(argv.keys()) if notFoundAttr: raise DistributiveError(_("The following attributes " "(%s) are not specified")\ %", ".join(map(lambda x:"DataPartition.%s"%x, notFoundAttr))) unnecessaryAttr = (set(dictDataPart.keys()) ^ set(argv.keys())) -\ set(dictDataPart.keys()) if unnecessaryAttr: raise DistributiveError(_("Failed to use attributes (%s)")\ %", ".join(map(lambda x:"DataPartition.%s"%x, unnecessaryAttr))) else: partObj = DataPartition() for attr, value in argv.items(): if attr in defaultAttr: continue setattr(partObj,attr,value) self.listPartitions.append(partObj) def getSystemId(self): """Get systemID for change [None,82,...]""" return map(lambda x: x.systemId, self.listPartitions) def getPartitionTable(self): """Get systemID for change [dos,gpt,...]""" return map(lambda x: x.partitionTable, self.listPartitions) def getIsFormat(self): """Get list is format [True,...]""" return map(lambda x: x.isFormat, self.listPartitions) def getFileSystems(self): """Get list file systems ["reiserfs",...]""" return map(lambda x: x.fileSystem, self.listPartitions) def getPartitions(self): """Get list partition ["/dev/sda",...]""" return map(lambda x: x.dev, self.listPartitions) def getMountPoints(self): """Get list mount point ["/boot",...]""" return map(lambda x: x.mountPoint, self.listPartitions) class PartitionDistributive(Distributive): formatUtilities = { 'ext2':'/sbin/mkfs.ext2 %s %s', 'ext3':'/sbin/mkfs.ext3 %s %s', 'ext4':'/sbin/mkfs.ext4 %s %s', 'jfs':'/sbin/mkfs.jfs %s -f %s', 'reiserfs':'/sbin/mkfs.reiserfs %s -f %s', 'btrfs':'/sbin/mkfs.btrfs %s %s', 'nilfs2':'/sbin/mkfs.nilfs2 %s %s', 'xfs':'/sbin/mkfs.xfs %s -f %s', 'vfat':'/usr/sbin/mkfs.vfat %s -F 32 %s', 'ntfs-3g':'/usr/sbin/mkfs.ntfs %s -FQ %s', 'ntfs':'/usr/sbin/mkfs.ntfs %s -FQ %s', 'swap':'/sbin/mkswap %s' } labelForUtilities = { 'ext2':'-L %s', 'ext3':'-L %s', 'ext4':'-L %s', 'btrfs':'-L %s', 'nilfs2':'-L %s', 'jfs':'-L %s', 'reiserfs':'-l %s', 'xfs':'-L %s', 'vfat':'-n %s', 'ntfs-3g':'-L %s', 'ntfs':'-L %s', } formatId = { 'ext2' : '83', 'ext3' : '83', 'ext4' : '83', 'reiserfs' : '83', 'btrfs' : '83', 'nilfs2' : '83', 'jfs' : '83', 'xfs' : '83', 'vfat' : '0b', 'swap' : '82' } formatIdGpt = { 'ext2' : '0700', 'ext3' : '0700', 'ext4' : '0700', 'reiserfs' : '0700', 'btrfs' : '0700', 'nilfs2' : '0700', 'jfs' : '0700', 'xfs' : '0700', 'vfat' : '0700', 'swap' : '8200' } def __init__(self,partition,parent=None,mdirectory="/mnt/calculate", check=False,multipartition=None,flagRemoveDir=True, fileSystem="reiserfs", isFormat=True,systemId=None, rootLabel="Calculate", partitionTable=None): """Initialize partition distributive mdirectory - directory for mount check - check partition name and raise DistributiveError if partition has bad name """ Distributive.__init__(self,parent=parent) self.partition = partition self.fileSystem = fileSystem self.mdirectory = mdirectory self.multipartition = multipartition self.flagRemoveDir = flagRemoveDir self.isFormat = isFormat self.DirectoryObject = None self.systemId = systemId self.rootLabel = rootLabel self.partitionTable = partitionTable def _mountPartition(self,partition,directory,opts=""): """Mount partition to directory""" self._makeDirectory(directory) if "ntfs" in opts: source_dir = isMount(partition) if source_dir: self._mountToBind(source_dir,directory) return self._mountToDirectory(partition,directory,opts) def _umountPartition(self,directory): """Umount partition and remove directory""" self._umountDirectory(directory) if self.flagRemoveDir: self._removeDirectory(directory) def releaseChild(self,child): """Umount child Directory distributive""" if isinstance(child,DirectoryDistributive): self._umountPartition(child.directory) child.directory = None def _mountBind(self,srcDirectory,destDirectory): """Mount directory to directory""" self._makeDirectory(destDirectory) self._makeDirectory(srcDirectory) self._mountToBind(srcDirectory,destDirectory) def postinstallMountBind(self): """Mount bind mount point and create mount dirs""" if self.multipartition and self.DirectoryObject: mulipartDataBind = filter(lambda x: x[2]=="bind", self.getMultipartData()) dirObj = self.DirectoryObject mdirectory = dirObj.directory for srcDir, destDir, fileSystem, isFormat, partTable \ in mulipartDataBind: realDestDir = pathJoin(mdirectory, destDir) realSrcDir = pathJoin(mdirectory, srcDir) self._mountBind(realSrcDir, realDestDir) isFormat = False partObj = PartitionDistributive(realSrcDir, flagRemoveDir=False, fileSystem=fileSystem, isFormat=isFormat, parent=dirObj) DirectoryDistributive(realDestDir,parent=partObj) def getMultipartData(self): """Get multipartition data""" mulipartData = zip(self.multipartition.getPartitions(), self.multipartition.getMountPoints(), self.multipartition.getFileSystems(), self.multipartition.getIsFormat(), self.multipartition.getPartitionTable()) return mulipartData def convertToDirectory(self): """Convert partition to directory by mounting""" mdirectory = self.mdirectory for child in self.childs: if isinstance(child,DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._mountPartition(self.partition,mdirectory) dirObj = DirectoryDistributive(mdirectory,parent=self) if self.multipartition: mulipartDataNotBind = filter(lambda x: x[2]!="bind", self.getMultipartData()) for dev, mountPoint, fileSystem, isFormat, partTable\ in mulipartDataNotBind: if fileSystem!="swap": realMountPoint = pathJoin(mdirectory, mountPoint) self._mountPartition(dev,realMountPoint,"-t %s"%fileSystem) partObj = PartitionDistributive(dev, flagRemoveDir=False, fileSystem=fileSystem, isFormat=isFormat, parent=dirObj) if fileSystem!="swap": DirectoryDistributive(realMountPoint,parent=partObj) self.DirectoryObject = dirObj return dirObj def formatAllPartitions(self): """Format all partitions""" FS,DEV,NEEDFORMAT,NEWID,PARTTABLE = 0,1,2,3,4 # get all information to matrix dataPartitions = zip(self.multipartition.getFileSystems() +\ [self.fileSystem], self.multipartition.getPartitions() + \ [self.partition], self.multipartition.getIsFormat() + \ [self.isFormat], self.multipartition.getSystemId() + \ [self.systemId], self.multipartition.getPartitionTable() + \ [self.partitionTable]) # get partition which need format formatPartitions = map(lambda x: (x[FS],x[DEV]), filter(lambda x: x[NEEDFORMAT] and x[FS]!="bind", dataPartitions)) # format all get partition for fileSystem, dev in formatPartitions: if fileSystem=="swap": self.formatSwapPartition(dev) else: if dev == self.partition: self.formatPartition(dev, format=fileSystem, label=self.rootLabel) else: self.formatPartition(dev, format=fileSystem) # change system id for partitions changeidPartitions = map(lambda x: (x[NEWID],x[DEV],x[PARTTABLE]), filter(lambda x: x[NEWID], dataPartitions)) for systemid, dev, partTable in changeidPartitions: self.changeSystemID(dev,systemid,partTable) return True def _checkMount(self,dev): """Checking mount point""" if isMount(dev): raise DistributiveError( _("Failed to format partition %s, because it is mounted")%dev) def formatPartition(self, dev,format="reiserfs",label=""): """Format partition""" if not format in self.formatUtilities: raise DistributiveError( _("Specified format of '%s' not supported")%format) if dev in map(lambda y: y.split(" ")[0], filter(lambda x: x.startswith("/"), open("/proc/swaps"))): raise DistributiveError(\ _("Failed to format partition %s, because it is used as swap")% dev) self._checkMount(dev) if not os.access(dev,os.W_OK): raise DistributiveError(_("Failed to format the partition") + " %s:\n%s"%(dev,_("Permission denied"))) if label: labelStr = self.labelForUtilities.get(format,"") if labelStr: labelStr = labelStr%label else: labelStr = "" execStr = self.formatUtilities[format]%(labelStr,dev) res,errmes = self.runOsCommand(execStr) if res == 0: return True else: raise DistributiveError(_("Failed to format the partition") + " %s:\n%s"%(dev,errmes)) def performFormat(self): """Perform format for all partition of this distributive""" if self.multipartition: self.formatAllPartitions() elif self.isFormat: self.formatPartition(self.partition,format=self.fileSystem, label=self.rootLabel) if self.systemId: self.changeSystemID(self.partition,self.systemId, self.partitionTable) def changeSystemID(self,dev,systemid,parttable): """Change partition id, specified by systemid""" deviceName = detectDeviceForPartition(dev) if deviceName is None: raise DistributiveError( _("Failed to determine the parent device for %s")%dev) # device hasn't any partition elif deviceName == "": return True fdiskProg, gdiskProg = checkUtils('/sbin/fdisk','/usr/sbin/gdisk') partitionNumber = \ getUdevDeviceInfo(name=dev).get('ID_PART_ENTRY_NUMBER','') devicePartitionCount = countPartitions(deviceName) if deviceName and not partitionNumber: raise DistributiveError( _("Failed to determine the partition number for %s")%dev) if parttable == "dos": fdisk = process(fdiskProg,deviceName,stderr=STDOUT) pipe = Popen([fdiskProg,deviceName], stdin=PIPE, stdout=PIPE,stderr=PIPE) if devicePartitionCount > 1: pipe.stdin.write("t\n%s\n%s\nw\n"%(partitionNumber, systemid)) else: pipe.stdin.write("t\n%s\nw\n"%systemid) pipe.stdin.close() pipe.wait() elif parttable == "gpt": pipe = Popen([gdiskProg,deviceName], stdin=PIPE, stdout=PIPE,stderr=PIPE) if devicePartitionCount > 1: pipe.stdin.write("t\n%s\n%s\nw\ny\n"%(partitionNumber, systemid)) else: pipe.stdin.write("t\n%s\nw\ny\n"%systemid) pipe.stdin.close() pipe.wait() for waittime in (0.1,0.2,0.5,1,2,4): if path.exists(dev): return True else: sleep(waittime) raise DistributiveError( _("Failed to found partition %s after changing the system id")%dev) def formatSwapPartition(self, dev): """Format swap partition""" if dev in map(lambda y: y.split(" ")[0], filter(lambda x: x.startswith("/"), open("/proc/swaps"))): raise DistributiveError(\ _("Failed to execute 'mkswap %s', the swap partition is used " "by the current system") %dev) if isMount(dev): raise DistributiveError( _("Failed to format partition %s, because it is mounted") %dev) execStr = self.formatUtilities["swap"]%dev res,errmes = self.runOsCommand(execStr) if res == 0: return True else: raise DistributiveError(_("Failed to execute '%s'")%execStr +\ "\n%s" %errmes) def installFrom(self, source): """Install distributive to partition from source distributive""" # get currect partition as directory distrTo = self.convertToDirectory() # install into directroy distributive from source distrTo.installFrom(source) class ArchiveDistributive(Distributive): def __init__(self,file,parent=None,mdirectory="/var/calculate/tmp/stage"): Distributive.__init__(self,parent=parent) self.file = file self.mdirectory = mdirectory def _detectArchive(self,file): """Detect archive by "/usr/bin/file" command Return bzip2,gzip,7z or None """ res,mes = self.runOsCommand("/usr/bin/file %s"%file) if "bzip2 compressed data" in mes: return "bzip2" elif "gzip compressed data" in mes: return "gzip" elif "7-zip archive data" in mes: return "7z" elif file and file.endswith(".tar.lzma"): if path.exists('/usr/bin/7za'): return "7z" else: return "lzma" return None def _unpackArchive(self,archfile,directory): """Unpack archive""" # archive is exists if not path.exists(archfile): raise DistributiveError(_("File '%s' not found")%archfile) # detect type archive archiveType = self._detectArchive(archfile) # make directory if archive was detected normally if archiveType: self._makeDirectory(directory) # unpack archive if archiveType == "7z": res,mes = self.runOsCommand("7za x -so %s | tar xf - -C %s/"% (archfile,directory)) elif archiveType == "lzma": res,mes = self.runOsCommand("lzma -dc %s | tar xf - -C %s/"% (archfile,directory)) elif archiveType == "bzip2": res,mes = self.runOsCommand("tar xjf %s -C %s/"% (archfile,directory)) elif archiveType == "gzip": res,mes = self.runOsCommand("tar xf %s -C %s/"% (archfile,directory)) else: raise DistributiveError(_("Unknown archive type by '%s'")% archfile) if res != 0: raise DistributiveError(_("Error during unpacking\n%s")%mes) def unpackTo(self,directory): """Unpack currect archive to directory""" self._unpackArchive(self.file,directory) def convertToDirectory(self): """Get archive as directory (unpack to directory)""" # check may be the archive already unpacked mdirectory = self.mdirectory for child in self.childs: if isinstance(child,DirectoryDistributive) and \ mdirectory in child.directory: return child # get temporary directory for unpacking mdirectory = self._getMntDirectory(mdirectory) dirdist = DirectoryDistributive(mdirectory,parent=self) self._unpackArchive(self.file,mdirectory) return dirdist def releaseChild(self,child): """Remove child Directory distributive""" if isinstance(child,DirectoryDistributive): self._removeDirectory(child.directory) child.directory = None def packToArchive(self,directory,file): res,errmes = self.runOsCommand("tar cf %s -C %s ."%(file,directory)) if res != 0: raise DistributiveError(_("Failed to create the archive") + " '%s':\n%s"%(file,errmes)) def installFrom(self, source): """Install distributive to partition from source distributive""" # get source distributive as directory distributive dFrom = source.convertToDirectory() # install into directroy distributive from source self.packToArchive(dFrom.directory, self.file) class SquashDistributive(Distributive): def __init__(self,file,parent=None,mdirectory="/mnt/livecd",exclude=None, compress=""): Distributive.__init__(self,parent=parent) self.file = file self.mdirectory = mdirectory self.exclude = [] if not exclude else exclude self.compress = compress \ if compress and compress != "gzip" else \ "" def _mountSquash(self,file,directory): """Mount squashfs to directory""" self._makeDirectory(directory) self._mountToDirectory(file,directory,mountopts="-o loop -t squashfs") def _umountSquash(self,directory): self._umountDirectory(directory) self._removeDirectory(directory) def convertToDirectory(self): mdirectory = self.mdirectory for child in self.childs: if isinstance(child,DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._mountSquash(self.file,mdirectory) return DirectoryDistributive(mdirectory,parent=self) def releaseChild(self,child): """Umount child Directory distributive""" if isinstance(child,DirectoryDistributive): self._umountSquash(child.directory) child.directory = None def packToSquash(self,directory,file): mksquashfsUtil = '/usr/bin/mksquashfs' if not path.exists(mksquashfsUtil): raise DistributiveError(_("Failed to create squash") + " : %s"%_("command '%s' not found")%mksquashfsUtil) cmd = [mksquashfsUtil, "%s/"%directory,file, "-no-progress"] if self.compress: cmd += ["-comp",self.compress] if self.exclude: cmd += ["-e"] + self.exclude processMkSquash = process(*cmd) if processMkSquash.failed(): raise DistributiveError(_("Failed to create squashfs") + " '%s':\n%s"%(file,processMkSquash.read())) def installFrom(self, source): """Install distributive to partition from source distributive""" # get source distributive as directory distributive dFrom = source.convertToDirectory() # install into directroy distributive from source self.packToSquash(dFrom.directory, self.file) class IsoDistributive(Distributive): def __init__(self,file,parent=None,mdirectory="/mnt/cdrom", bdirectory="/var/calculate/tmp/iso",exclude=None, compress="gzip", volid="CALCULATE"): Distributive.__init__(self,parent=parent) self.file = file self.volid = volid if path.isdir(self.file): self.mdirectory = self.file else: self.mdirectory = mdirectory if file == bdirectory: self.bdirectory = file else: self.bdirectory = self._getMntDirectory(bdirectory) self.exclude = [] if not exclude else exclude self.compress = compress def probe(self): """Check directory for iso content""" try: pathname = self.getIsoContentDirectory() except: return False return path.exists(path.join(pathname,"syslinux")) and \ path.exists(path.join(pathname,"isolinux")) def _mountIso(self,file,directory): if self.file != self.mdirectory: self._makeDirectory(directory) tf = typeFile(magic=0x6|0x20) ftype = tf.getMType(file) if "block special" in ftype: mopts = "" else: mopts = "-o loop" self._mountToDirectory(file,directory,mountopts=mopts) def _umountIso(self,directory): if self.file != self.mdirectory: self._umountDirectory(directory) self._removeDirectory(directory) def convertToSquash(self): mdirectory = self.mdirectory for child in self.childs: if isinstance(child,SquashDistributive) and \ mdirectory in child.file: return child if self.mdirectory != self.file: mdirectory = self._getMntDirectory(mdirectory) self._mountIso(self.file,mdirectory) fileLive = self._getLastLive(mdirectory) if not fileLive: self._umountIso(mdirectory) raise DistributiveError(_("Iso %s contains no live image") % self.file) return SquashDistributive(path.join(mdirectory,fileLive), parent=self,exclude=self.exclude, compress=self.compress) def getIsoContentDirectory(self): """Return directory with content of iso image""" squash = self.convertToSquash() return path.dirname(squash.file) def releaseChild(self,child): """Umount child Directory distributive""" if isinstance(child,SquashDistributive): self._umountIso(path.dirname(child.file)) child.directory = None def convertToDirectory(self): return self.convertToSquash().convertToDirectory() def packToIso(self,directory,file): # remove previous version of iso try: if path.exists(file): os.unlink(file) except Exception, e: raise DistributiveError(_("Failed to remove") +\ " %s:\n%s"%(file,str(e))) except KeyboardInterrupt, e: self.setSignalInterrupt() raise DistributiveError(_("Failed to remove") +\ " %s:\n%s"%(file,str(e))) mkisoUtil = '/usr/bin/mkisofs' if not path.exists(mkisoUtil): raise DistributiveError(_("Failed to create the iso image") + " : %s"%_("command '%s' not found")%mkisoUtil) efiImage = "boot/grub/efi.img" if os.path.exists(os.path.join(directory,efiImage)): params = ["-J","-R","-l","-no-emul-boot","-boot-load-size 4", "-udf","-boot-info-table","-b isolinux/isolinux.bin", "-V %s"%self.volid, "-c isolinux/boot.cat","-eltorito-alt-boot", "-no-emul-boot","-eltorito-platform efi", "-eltorito-boot %s"%efiImage] else: params = ["-b isolinux/isolinux.bin","-no-emul-boot", "-V %s"%self.volid, "-boot-load-size 4","-boot-info-table","-iso-level 4", "-hide boot.catalog"] res,errmes = self.runOsCommand( "%(progname)s %(params)s -o %(target)s %(source)s/"% {'progname':mkisoUtil, 'params':" ".join(params), 'target':file, 'source':directory}) if res == 0: return True else: raise DistributiveError(_("Failed to create the iso image") + " %s:\n%s"%(file,errmes)) def prepareIso(self): raise DistributiveError( _("Iso image cannot be created without overriding prepareIso")) def installFrom(self, source): """Install distributive to partition from source distributive""" # make temporary directory for creating iso image isoDirectory = self.bdirectory self._makeDirectory(isoDirectory) try: # getting squash from source liveimage = self._getLastLive(isoDirectory) if liveimage: curnum = self._getSquashNum(self.reLive.search(liveimage)) liveimage = "livecd.squashfs.%d"%(curnum+1) else: liveimage = "livecd.squashfs" if isinstance(source,SquashDistributive): self._copyfile(source.file, path.join(isoDirectory,liveimage)) else: distDirectory = source.convertToDirectory() squashDistr = SquashDistributive( path.join(isoDirectory,liveimage), exclude=self.exclude, compress=self.compress) squashDistr.installFrom(distDirectory) # prepare iso self.prepareIso(isoDirectory) # pack iso if self.bdirectory != self.file: self.packToIso(isoDirectory, self.file) except DistributiveError,e: raise e except KeyboardInterrupt,e: self.setSignalInterrupt() raise DistributiveError(_("Keyboard interruption")) def close(self): # close all child Distributive.close(self) # remove temporary directory if path.lexists(self.bdirectory) and self.file != self.bdirectory: self._removeDirectory(self.bdirectory) class FlashDistributive(PartitionDistributive): def _checkMount(self,dev): """Checking mount point""" mp = isMount(dev) if mp: if mp.startswith('/media'): self._umountDirectory(mp) else: raise DistributiveError( _("Failed to format partition %s, because it is mounted")%dev) def installFrom(self, source): """Install distributive to partition from source distributive""" # make temporary directory for creating iso image distrTo = self.convertToDirectory() # getting squash from source if isinstance(source,IsoDistributive): self.rsync(source.getIsoContentDirectory(),distrTo.directory) else: raise DistributiveError( _("Installation to flash does not support %s"% source.__class__.__name__)) class PxeDistributive(Distributive): def __init__(self,directory,parent=None): Distributive.__init__(self,parent=parent) self.directory = path.join(directory,"calculate") self.origdir = directory def getDirectory(self): return self.origdir def installFrom(self,source): # make temporary directory for creating iso image distrTo = self.directory # getting squash from source if isinstance(source,IsoDistributive): if path.exists(self.directory): removeDir(self.directory) self._makeDirectory(self.directory) self.rsync(source.getIsoContentDirectory(),distrTo) else: raise DistributiveError( _("Installation for PXE does not support %s"% source.__class__.__name__)) class ScratchDistributive(Distributive): def __init__(self,directory,parent=None,mdirectory="/mnt/install", check=False): Distributive.__init__(self,parent=parent) self.directory = directory self.mdirectory = mdirectory if check and not (path.exists(path.join(directory,"workspace")) and \ path.exists(path.join(directory,"delta")) and \ path.exists(path.join(directory,"calculate"))): raise DistributiveError( _("Wrong scratch distribution in '%s'")%directory) def _mountLayers(self,source,target): """Mount squashfs to directory""" self._makeDirectory(target) self._mountToDirectory("none",target, mountopts="-t aufs "+\ "-o udba=reval,br:%(work)s=rw:%(delta)s=ro+wh:%(static)s=ro" %\ {"work":path.join(source,"workspace"), "delta":path.join(source,"delta"), "static":path.join(source,"calculate")}) def _umountLayers(self,directory): self._umountDirectory(directory) self._removeDirectory(directory) def _mountLiveImage(self,file,directory): """Mount squashfs to directory""" self._makeDirectory(directory) self._mountToDirectory(file,directory,mountopts="-o loop -t squashfs") def _umountLiveImage(self,directory): self._umountDirectory(directory) def convertToDirectory(self): """Convert scrach directories to one directory""" mdirectory = self.mdirectory for child in self.childs: if isinstance(child,DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) liveFile = self._getLastLive(self.directory) self._mountLiveImage(path.join(self.directory,liveFile), path.join(self.directory,"calculate")) self._mountLayers(self.directory,mdirectory) return DirectoryDistributive(mdirectory,parent=self) def releaseChild(self,child): """Umount child Directory distributive""" if isinstance(child,DirectoryDistributive): self._umountLayers(child.directory) self._umountLiveImage(path.join(self.directory,"calculate")) child.directory = None def installFrom(self, source): """Install distributive to partition from source distributive""" # get source distributive as directory distributive dFrom = source.convertToDirectory() # install into directroy distributive from source self.packToSquash(dFrom.directory, self.file) class ScratchPartitionDistributive(PartitionDistributive): def releaseChild(self,child): """Umount child Directory distributive""" if isinstance(child,ScratchDistributive): self._umountPartition(child.directory) child.directory = None def convertToDirectory(self): """Convert scratch partition to directory by scratch directory""" return self.convertToScratch().convertToDirectory() def convertToScratch(self): mdirectory = self.mdirectory for child in self.childs: if isinstance(child,ScratchDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._mountPartition(self.partition,mdirectory) return ScratchDistributive(mdirectory,parent=self) def prepareScratch(self,directory): """Create need scratch directories""" for scrDirectory in ("calculate","delta","workspace"): self._makeDirectory(path.join(directory,scrDirectory)) def getBootDirectory(self): """Get directory which contains boot""" return path.join(self.convertToScratch().directory,"boot") def postinstallMountBind(self): pass def installFrom(self, source): """Install distributive to partition from source distributive""" # get currect partition as directory distrTo = self.convertToScratch() scratchDirectory = distrTo.directory try: # getting squash from source if isinstance(source,IsoDistributive): source = source.convertToSquash() if isinstance(source,SquashDistributive): self._copyfile(source.file, path.join(scratchDirectory,"livecd.squashfs")) else: distDirectory = source.convertToDirectory() squashDistr = SquashDistributive( path.join(scratchDirectory,"livecd.squashfs")) squashDistr.installFrom(distDirectory) # prepare scratch self.prepareScratch(scratchDirectory) except DistributiveError,e: raise e except KeyboardInterrupt,e: self.setSignalInterrupt() raise DistributiveError(_("Keyboard interruption"))