From d88237145e5dd401bd6e00ace0512027c162fa3e Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Tue, 19 Jun 2012 17:53:39 +0400 Subject: [PATCH] Move cl_distr from calculate-install --- pym/cl_distr.py | 1520 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1520 insertions(+) create mode 100644 pym/cl_distr.py diff --git a/pym/cl_distr.py b/pym/cl_distr.py new file mode 100644 index 0000000..220fac7 --- /dev/null +++ b/pym/cl_distr.py @@ -0,0 +1,1520 @@ +#-*- coding: utf-8 -*- + +# Copyright 2010 Calculate Ltd. http://www.calculate-linux.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import path +from random import choice +import string +import os +import types +from time import sleep +import re +import sys +import operator +from cl_utils import runOsCommand,isMount,removeDir,typeFile,pathJoin, \ + process,getRunCommands,getTupleVersion,cmpVersion, \ + detectDeviceForPartition, getProgPath,listDirectory, \ + checkUtils,STDOUT,getUdevDeviceInfo, countPartitions +from cl_vars_share import varsShare +from shutil import copyfile,copytree +from cl_template import _terms +from subprocess import Popen,PIPE,STDOUT +from cl_fill import fillVars +from cl_vars_share import varsShare + +import threading + +from cl_lang import lang +tr = lang() +tr.setLocalDomain('cl_install') +tr.setLanguage(sys.modules[__name__]) + +class SignalInterrupt: + __interruptProcessObjs = [] + __sigint = False + + def setSignalInterrupt(self): + """Handler "SIGINT""" + if SignalInterrupt.__sigint is False: + sys.stdout.write("\b\b") + # killed processes + while (SignalInterrupt.__interruptProcessObjs): + process = SignalInterrupt.__interruptProcessObjs.pop() + if hasattr(process, "pipe") and hasattr(process.pipe,"kill"): + process.pipe.kill() + SignalInterrupt.__sigint = True + + def getSignalInterrupt(self): + return SignalInterrupt.__sigint + + def addInterruptProcess(self, process): + SignalInterrupt.__interruptProcessObjs.append(process) + + +class Spinner(threading.Thread): + stopSignal = threading.Event() + sequence= map(lambda x:x,iter("/-\\|")) + write = sys.stdout.write + selfthread = None + + def setWriteFunc(self,writeFunc): + Spinner.write = writeFunc + + def init(self,checkFunction=None,interval=0.1): + self.curpos = 0 + self.checkFunction = checkFunction + self.interval = interval + Spinner.stopSignal.clear() + + def stop(self): + Spinner.stopSignal.set() + if Spinner.selfthread: + Spinner.selfthread.join(2) + Spinner.selfthread = None + #Spinner().write('\n') + Spinner().write('\b') + + def run(self): + self.write(self.sequence[-1]) + Spinner.selfthread = self + while not Spinner.stopSignal.isSet(): + if self.checkFunction and self.checkFunction(): + self.write("\b%s"%self.sequence[self.curpos]) + self.curpos += 1 + if self.curpos >= len(self.sequence): + self.curpos = 0 + sleep(self.interval) + +class DistributiveError(Exception): + """Error for distributive operations""" + pass + +class DistributiveRepository: + varsShare = varsShare() + + contentCache = {} + + marches = ['i686','x86_64'] + + extensiton = ['iso','tar.bz2','tar.gz','tar.7z','tar.lzma'] + + reDistName = re.compile(""" + ^.*/(?P%(name)s) + -(?P%(ver)s) + -(?P%(march)s) + .(?P%(ext)s)$""" % + {'name':"[a-z0-9]+", + 'ver':r"(\d+\.)*\d+", + 'march':"|".join(marches), + 'ext':"|".join(extensiton) + }, re.X) + + def __init__(self,directories=[]): + self.dirs = directories + + def ini_to_dict(self,filename): + """Convert ini values to dict""" + if os.access(filename,os.R_OK): + return dict(map(lambda x: x.strip().rpartition('=')[0::2], + open(filename,'r'))) + else: + return {} + + def _getfromcontent(self,filename,addFunc=None): + """Get info from content""" + origfilename = filename + if filename in DistributiveRepository.contentCache: + return DistributiveRepository.contentCache[filename].copy() + varsShare = self.varsShare + distr = None + # may be directory is isodir (directory which contains iso image) + extname = "isodir" + try: + distr = IsoDistributive(filename) + if not distr.probe() and filename.startswith('/dev'): + distr.close() + distr = None + distr = PartitionDistributive(filename) + filename = distr.convertToDirectory().directory + except Exception,e: + extname = "dir" + d = self.ini_to_dict(path.join(filename, + 'etc/calculate/calculate.ini')) + if not d or not "march" in d: + if path.exists(path.join(filename,'lib64')): + d['march'] = 'x86_64' + else: + d['march']= 'i686' + if d: + d['ext'] = extname + d["name"] = varsShare.getShortnameByMakeprofile(filename) or \ + varsShare.getShortnameByIni(filename) or \ + varsShare.detectOtherShortname(filename) or \ + "Linux" + d['ver'] = \ + varsShare.getVersionFromMetapackage(filename,d["name"]) or \ + varsShare.getVersionFromCalculateIni(filename) or "0" + reOsLinuxBuild = re.compile("^os_linux_build\s*=\s*(\S+)\s*$") + os_linux_build = \ + map(lambda x:x.groups()[0], + filter(lambda x:x, + map(reOsLinuxBuild.search, + reduce(lambda x,y:x+y, + map(lambda x:open(x,"r").readlines(), + filter(path.exists, + [path.join(filename,"etc/calculate/calculate2.env")])),[])))) + if os_linux_build: + d['build'] = os_linux_build[-1] + else: + d['build'] = "" + if addFunc: + d = addFunc(filename,d) + if distr: + distr.close() + DistributiveRepository.contentCache[origfilename] = d + return d.copy() + + def _getdistrinfo(self,filename): + """Get information by distributive""" + varsShare = self.varsShare + # if filename is directory + if not path.isfile(filename): + return self._getfromcontent(filename) + else: + match = self.reDistName.match(filename) + if not match: + return {} + distdic = match.groupdict() + distdic["build"] = "" + if "ver" in distdic: + if re.match("^\d{8}$", distdic["ver"]): + distdic["build"] = distdic["ver"] + distdic["ver"] = "" + return distdic + + def _getAvailableShortnames(self): + """Get available distributives shortnames""" + distros = filter(lambda x:x, + map(self.reDistName.search,self._getAvailableDistributives())) + return sorted(list(set(map(lambda x:x.groupdict()['name'],distros)))) + + def _getAvailableDistributives(self,system=None,shortname=None,march=None, + version=None,op_compare=None): + """Get all distributives by filter""" + if op_compare is None: + op_compare = operator.eq + if version: + version = getTupleVersion(version) + def distfilter(dist): + d = self._getdistrinfo(dist) + if not d: + return False + # check filter conditions + if system and self.system(d['name']) != system: + return False + if not "name" in d or not "ver" in d: + return False + if shortname and d['name'].lower() != shortname.lower(): + return False + if march and d['march'] != march: + return False + if version and not op_compare(getTupleVersion(d['ver']), version): + return False + return True + + def listdistr(pathname): + if path.exists(path.join(pathname,'etc/make.profile')): + return [pathname] + elif path.exists(path.join(pathname,'livecd')): + return [pathname] + elif pathname.startswith('/dev/'): + return [pathname] + else: + # discard inner directories + return filter(lambda x:not path.isdir( path.join(pathname,x)), + listDirectory(pathname)) + # get lists files in directories + allFiles = map(lambda x: map(lambda y: path.join(x,y), + listdistr(x)), + # discard not exists distrib directories + filter(lambda x: os.access(x,os.R_OK), + self.dirs)) + # filter distributives + return filter(distfilter, + # join files lists to one list + reduce(lambda x,y: x + y, + allFiles, [])) + + def getDistributiveByFile(self,filename): + """Get Distributive object by filename""" + # MAGIC_COMPRESS 0x000004 Check inside compressed files + tf = typeFile(magic=0x6) + ftype = tf.getMType(filename) + if "block special" in ftype: + return IsoDistributive(filename) + if "ISO 9660 CD-ROM" in ftype: + return IsoDistributive(filename) + elif "7-zip" in ftype or \ + "POSIX tar archive" in ftype: + return ArchiveDistributive(filename) + elif "Squashfs filesystem" in ftype: + return SquashDistributive(filename) + elif path.isdir(filename): + if path.exists(path.join(filename,"livecd")): + return IsoDistributive(filename) + else: + return DirectoryDistributive(filename) + else: + raise DistributiveError(_("Wrong distribution") + " '%s':\n%s"\ + %(filename,ftype)) + + def extcomparator(self,*exts): + avexts = ("iso","isodir","dir") + vals = [] + for i in exts: + if i in avexts: + vals.append(avexts.index(i)) + else: + vals.append(-1) + return cmp(vals[0],vals[1]) + + term = _terms() + + def sortdistrfunc(self,x,y): + """Func of comparing two distributive""" + ver1, ver2 = x[1].get('ver',""), y[1].get('ver',"") + if ver1 and ver2 and ver1 != "0" and ver2 != "0" and ver1 != ver2: + return cmpVersion(ver1,ver2) + build1,build2 = self.term._convertVers(x[1].get('build',""), + y[1].get('build',"")) + if build1 != build2: + return cmp(build1,build2) + else: + ext1 = x[1].get('ext',"") + ext2 = y[1].get('ext',"") + return self.extcomparator(ext1,ext2) + + def getBestDistributive(self,system=None,shortname=None,march=None, + version=None,discardType=[], op_compare=None): + """Get the actualest distributive""" + availDistrs = self._getAvailableDistributives(system,shortname, + march,version, + op_compare) + availDistrs = filter(lambda x:x[1] and "ext" in x[1] and + not x[1]["ext"] in discardType, + map(lambda x:(x,self._getdistrinfo(x)), + availDistrs)) + availDistrs = sorted(availDistrs,self.sortdistrfunc,reverse=True) + + if availDistrs: + return availDistrs[0][0] + else: + return None + + def _findLatestFile(self,dirs,reMatch,keyfunc): + """Find latest file in dirs, which match by reMatch, + comparable part get by keyfunc""" + existsdirs = filter(path.exists,dirs) + listimgs = reduce(lambda x,y:x + \ + map(lambda x:reMatch.search(path.join(y,x)), + listDirectory(y)), + existsdirs,[]) + listimgs = filter(lambda x:x, listimgs) + if listimgs: + return max(listimgs,key=keyfunc).group() + return "" + + def getBestStage(self,march=None,hardened=None): + """Get latest stage by march""" + if march: + march = {'x86_64':'amd64'}.get(march,march) + else: + march = "[^-]+" + if hardened is None: + hardened = "(?:-hardened)?" + elif hardened == True: + hardened = "-hardened" + elif hardened == False: + hardened = "" + reStage = re.compile(r'^.*/stage3-%s%s-(\d+)\.tar\.bz2$'% + (march,hardened),re.S) + return self._findLatestFile(self.dirs,reStage,lambda x:x.groups()[0]) + +class Distributive(object, SignalInterrupt): + """Distributive object. Parent object for all distributive.""" + mountError = _("Failed to mount") + " %s:\n%s" + reLive = re.compile(r"^live[^.]*\.squashfs(\.(\d+))?$",re.S) + flagSpinner=True + + def __init__(self, parent=None): + self.childs = [] + # if specified parent + if parent: + # save parent type for correct resource release + self.parent = parent + # append this object as child to specified parent + parent.childs.append(self) + else: + self.parent = None + + def detach(self): + """Detach child distributive from parent. + + At example: ArchiveDistributive create child DirectoryDistributive by + unpacking to temporary directory and at close method remove it. If the + removing directroy do not need, then need call detach in + DirectoryDistributive object + + dist1 = ArchiveDistributive(file="/os.tar.bz2",mdirectory="/tmp/test") + dist2 = dist1.convertToDirectory() + dist2.detach() + dist1.close() + ... + """ + self.parent = None + + def close(self): + """Release all child distributive and release himself. + + Need call this method at end work with object for right releasing + resources. + + Example: + dist1 = PartitionDistributive(partition="/dev/sda2") + dist2 = dist1.convertToDirectory() + dist1.close() + """ + # close all child + if self.childs: + for child in reversed(self.childs): + # check detach + if child.parent: + child.close() + self.childs = [] + # if has parent + if self.parent: + self.parent.releaseChild(self) + self.parent = None + + def releaseChild(self,child): + """Method of release child state of distributive + + At example: PartitionDistributive may be DirectoryDistributive by + mounting it to directory. But at end this directory must be + unmounted.""" + pass + + def convertToDirectory(self): + """Default c raise error about impossible convert object""" + raise DistributiveError(_("Failed to convert") + " '%s' "\ + %self.__class__.__name__ + _("to") +\ + " '%s'" %"DirectoryDistributive") + +# def __del__(self): +# """Uncomment this method for automaticaly release all distributive +# instance""" +# self.close() + + def runOsCommand(self, *argv, **kwarg): + res, mes = runOsCommand(*argv,**kwarg) + mes = "\n".join(map(lambda x: x.strip(), mes)) + return res,mes + + def getDirectory(self): + """Get directory which contains distro""" + return self.convertToDirectory().directory + + def getBootDirectory(self): + """Get directory which contains boot""" + return path.join(self.getDirectory(),"boot") + + def _makeDirectory(self,pathname): + """Make directory and parent. + + If directory exists then return False else True""" + try: + parent = path.split(path.normpath(pathname))[0] + if not path.exists(parent): + self._makeDirectory(parent) + else: + if path.exists(pathname): + return False + os.mkdir(pathname) + return True + except Exception, e: + raise DistributiveError(_("Failed to create the directory") + + " '%s':\n%s" %(pathname,str(e))) + except KeyboardInterrupt, e: + self.setSignalInterrupt() + raise DistributiveError(_("Failed to create the directory") + + " '%s':\n%s" %(pathname,str(e))) + + def _removeDirectory(self,directory): + """Remove directory and files contained in it""" + try: + removeDir(directory) + except Exception, e: + raise DistributiveError(_("Failed to remove the directory from") +\ + " '%s':\n%s" %(directory,str(e))) + except KeyboardInterrupt, e: + self.setSignalInterrupt() + raise DistributiveError(_("Failed to remove the directory from") +\ + " '%s':\n%s" %(directory,str(e))) + + def _copyfile(self,infile,outfile): + try: + copyfile(infile,outfile) + except (Exception),e: + raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s"\ + %(infile,outfile,str(e))) + + def _copytree(self,indir,outdir): + try: + copytree(indir,outdir,symlinks=True) + except Exception ,e: + raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s"\ + %(indir,outdir,str(e))) + except KeyboardInterrupt, e: + self.setSignalInterrupt() + raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s"\ + %(indir,outdir,str(e))) + + def rsync(self,fromdir,todir,hideSpin=False): + """Copy files from 'fromdir' directory to 'todir' directory""" + if self.flagSpinner and not hideSpin: + def checkRsync(): + try: + return len(filter(lambda x:"rsync\x00-a\x00-H\x00-x" in x, + getRunCommands()))>2 + except: + return False + spin = Spinner() + spin.init(checkFunction=checkRsync,interval=0.6) + spin.start() + rsyncCmd = varsShare().getProgPath('/usr/bin/rsync') + if not rsyncCmd: + raise DistributiveError(_("Failed to find '%s' command")%"rsync") + try: + rsyncProcess = process(rsyncCmd, "-a","-H", "-x", + "%s/"%fromdir,"%s/"%todir,stderr=STDOUT) + self.addInterruptProcess(rsyncProcess) + res = rsyncProcess.success() + errmes = rsyncProcess.read() + except Exception,e: + res = False + errmes = str(e) + except KeyboardInterrupt: + self.setSignalInterrupt() + if self.flagSpinner: + spin.stop() + raise KeyboardInterrupt() + if self.flagSpinner and not hideSpin: + spin.stop() + if res is True: + return True + else: + if "No space left on device" in errmes: + errmes = "No space left on device" + raise DistributiveError(_("Failed to copy files from") +\ + " '%s' " %fromdir + _("to") +\ + " '%s':\n%s" %(todir,errmes)) + + def _mountToBind(self,srcDirectory,destDirectory): + """Mount srcDirectory to destDirectory""" + mount = process('/bin/mount',"-o","bind",srcDirectory,destDirectory, + stderr=STDOUT) + if mount.success(): + return True + else: + try: + os.rmdir(destDirectory) + except: + pass + raise DistributiveError(self.mountError%(srcDirectory,mount.read())) + + def performFormat(self): + pass + + def formatPartition(self,format=""): + pass + + def rndString(self): + """Get random string with len 8 char""" + return "".join([choice(string.ascii_letters+string.digits) + for i in xrange(0,8)]) + + def _getSquashNum(self,reMatch): + if reMatch.groups()[1] and reMatch.groups()[1].isdigit(): + return int(reMatch.groups()[1]) + else: + return 0 + + def _getLastLive(self,directory): + """Get last live squashfs image from directory""" + squashfiles = filter(lambda x:x, + map(self.reLive.search, + listDirectory(directory))) + if squashfiles: + return max(squashfiles, key=self._getSquashNum).group() + return None + + + def _mountToDirectory(self,file,directory,mountopts="",count=2): + """Mount squashfs to directory""" + NO_SUCH_DEVICE = 2816 + if isMount(directory): + raise DistributiveError(_("Failed to mount to the directory: %s\n")\ + %directory+ _("Directory already mounted")) + mountopts = filter(lambda x:x, + mountopts.split(" ")) + mountProcess = process('/bin/mount',file,directory,*mountopts, + stderr=STDOUT) + if mountProcess.success(): + return True + else: + #2816 code return by mount if device is absent (update /dev by udev) + # try mount 3 times with interval 0.5 second + if mountProcess.returncode() == NO_SUCH_DEVICE and count: + sleep(0.5) + mountProcess.close() + return self._mountToDirectory(file,directory,mountopts,count-1) + try: + self._removeDirectory(directory) + except: + pass + raise DistributiveError(self.mountError%(file,mountProcess.read())) + + def _umountDirectory(self,directory): + """Umount directory""" + if isMount(directory): + for wait in [0,0.5,2,5]: + sleep(wait) + processUmount = process('/bin/umount',directory,stderr=STDOUT) + if processUmount.success(): + return True + raise DistributiveError(_("Failed to umount") + " %s:\n%s"% + (directory,processUmount.read())) + else: + return True + + def _getMntDirectory(self,directory): + """Get directory name, which will use for mounting or unpacking + + If queried name is not free then to name append random string + """ + newDirectoryName = directory + while path.exists(newDirectoryName): + newDirectoryName = "%s.%s"%(directory,self.rndString()) + return newDirectoryName + +class DirectoryDistributive(Distributive): + def __init__(self,directory,parent=None,mdirectory=None): + Distributive.__init__(self,parent=parent) + self.directory = directory + self.mdirectory = mdirectory + if not parent: + self._makeDirectory(self.directory) + + def bindDirectory(self,mdirectory): + for child in self.childs: + if isinstance(child,DirectoryDistributive) and \ + mdirectory in child.directory: + return child + mdirectory = self._getMntDirectory(mdirectory) + self._makeDirectory(mdirectory) + self._mountToBind(self.directory,mdirectory) + return DirectoryDistributive(mdirectory,parent=self) + + def releaseChild(self,child): + """Remove child Directory distributive""" + if isinstance(child,DirectoryDistributive): + self._umountDirectory(child.directory) + self._removeDirectory(child.directory) + child.directory = None + + def convertToDirectory(self): + if self.mdirectory: + return self.bindDirectory(self.mdirectory) + else: + return self + + def performFormat(self): + """Format for directory - removing all files""" + execStr = '/bin/rm -rf --one-file-system %s'%self.directory + res,errmes = self.runOsCommand(execStr) + if res == 0: + return True + else: + raise DistributiveError(_("Failed to format the partition") + + " %s:\n%s"%(dev,errmes)) + self._makeDirectory(self.directory) + + def installFrom(self, source): + """Install distributive to directory from source distributive""" + if isinstance(source,ArchiveDistributive): + source.unpackTo(self.directory) + else: + # get source distributive as directory distributive + dFrom = source.convertToDirectory() + # copy distributive from source to this + self.rsync(dFrom.directory,self.directory) + +class DataPartition: + """Data partition""" + dev = None + mountPoint = None + fileSystem = "reiserfs" + isFormat = False + systemId = None + partitionTable = None + +class MultiPartitions: + """Data partition list""" + listPartitions = [] + + def addPartition(self, **argv): + """Add partition in data partition list""" + dictDataPart = reduce(lambda x,y:\ + x.update({y:getattr(DataPartition,y)}) or x, + filter(lambda x: not x.startswith('_'), + DataPartition.__dict__),{}) + updateAttrData = filter(lambda x: x[1]!=None, dictDataPart.items()) + defaultAttr = [] + for attrName, attrValue in updateAttrData: + if not attrName in argv.keys(): + defaultAttr.append(attrName) + argv[attrName] = attrValue + if set(argv.keys()) != set(dictDataPart.keys()): + notFoundAttr = set(dictDataPart.keys()) - set(argv.keys()) + if notFoundAttr: + raise DistributiveError(_("The following attributes " + "(%s) are not specified")\ + %", ".join(map(lambda x:"DataPartition.%s"%x, notFoundAttr))) + unnecessaryAttr = (set(dictDataPart.keys()) ^ set(argv.keys())) -\ + set(dictDataPart.keys()) + if unnecessaryAttr: + raise DistributiveError(_("Failed to use attributes (%s)")\ + %", ".join(map(lambda x:"DataPartition.%s"%x, unnecessaryAttr))) + else: + partObj = DataPartition() + for attr, value in argv.items(): + if attr in defaultAttr: + continue + setattr(partObj,attr,value) + self.listPartitions.append(partObj) + + def getSystemId(self): + """Get systemID for change [None,82,...]""" + return map(lambda x: x.systemId, self.listPartitions) + + def getPartitionTable(self): + """Get systemID for change [dos,gpt,...]""" + return map(lambda x: x.partitionTable, self.listPartitions) + + def getIsFormat(self): + """Get list is format [True,...]""" + return map(lambda x: x.isFormat, self.listPartitions) + + def getFileSystems(self): + """Get list file systems ["reiserfs",...]""" + return map(lambda x: x.fileSystem, self.listPartitions) + + def getPartitions(self): + """Get list partition ["/dev/sda",...]""" + return map(lambda x: x.dev, self.listPartitions) + + def getMountPoints(self): + """Get list mount point ["/boot",...]""" + return map(lambda x: x.mountPoint, self.listPartitions) + + +class PartitionDistributive(Distributive): + formatUtilities = { 'ext2':'/sbin/mkfs.ext2 %s %s', + 'ext3':'/sbin/mkfs.ext3 %s %s', + 'ext4':'/sbin/mkfs.ext4 %s %s', + 'jfs':'/sbin/mkfs.jfs %s -f %s', + 'reiserfs':'/sbin/mkfs.reiserfs %s -f %s', + 'btrfs':'/sbin/mkfs.btrfs %s %s', + 'nilfs2':'/sbin/mkfs.nilfs2 %s %s', + 'xfs':'/sbin/mkfs.xfs %s -f %s', + 'vfat':'/usr/sbin/mkfs.vfat %s -F 32 %s', + 'ntfs-3g':'/usr/sbin/mkfs.ntfs %s -FQ %s', + 'ntfs':'/usr/sbin/mkfs.ntfs %s -FQ %s', + 'swap':'/sbin/mkswap %s' + } + labelForUtilities = { 'ext2':'-L %s', + 'ext3':'-L %s', + 'ext4':'-L %s', + 'btrfs':'-L %s', + 'nilfs2':'-L %s', + 'jfs':'-L %s', + 'reiserfs':'-l %s', + 'xfs':'-L %s', + 'vfat':'-n %s', + 'ntfs-3g':'-L %s', + 'ntfs':'-L %s', + } + + formatId = { 'ext2' : '83', + 'ext3' : '83', + 'ext4' : '83', + 'reiserfs' : '83', + 'btrfs' : '83', + 'nilfs2' : '83', + 'jfs' : '83', + 'xfs' : '83', + 'vfat' : '0b', + 'swap' : '82' + } + + formatIdGpt = { 'ext2' : '0700', + 'ext3' : '0700', + 'ext4' : '0700', + 'reiserfs' : '0700', + 'btrfs' : '0700', + 'nilfs2' : '0700', + 'jfs' : '0700', + 'xfs' : '0700', + 'vfat' : '0700', + 'swap' : '8200' + } + + def __init__(self,partition,parent=None,mdirectory="/mnt/calculate", + check=False,multipartition=None,flagRemoveDir=True, + fileSystem="reiserfs", isFormat=True,systemId=None, + rootLabel="Calculate", partitionTable=None): + """Initialize partition distributive + + mdirectory - directory for mount + check - check partition name and raise DistributiveError if partition + has bad name + """ + Distributive.__init__(self,parent=parent) + self.partition = partition + self.fileSystem = fileSystem + self.mdirectory = mdirectory + self.multipartition = multipartition + self.flagRemoveDir = flagRemoveDir + self.isFormat = isFormat + self.DirectoryObject = None + self.systemId = systemId + self.rootLabel = rootLabel + self.partitionTable = partitionTable + + def _mountPartition(self,partition,directory,opts=""): + """Mount partition to directory""" + self._makeDirectory(directory) + if "ntfs" in opts: + source_dir = isMount(partition) + if source_dir: + self._mountToBind(source_dir,directory) + return + self._mountToDirectory(partition,directory,opts) + + def _umountPartition(self,directory): + """Umount partition and remove directory""" + self._umountDirectory(directory) + if self.flagRemoveDir: + self._removeDirectory(directory) + + def releaseChild(self,child): + """Umount child Directory distributive""" + if isinstance(child,DirectoryDistributive): + self._umountPartition(child.directory) + child.directory = None + + def _mountBind(self,srcDirectory,destDirectory): + """Mount directory to directory""" + self._makeDirectory(destDirectory) + self._makeDirectory(srcDirectory) + self._mountToBind(srcDirectory,destDirectory) + + def postinstallMountBind(self): + """Mount bind mount point and create mount dirs""" + if self.multipartition and self.DirectoryObject: + mulipartDataBind = filter(lambda x: x[2]=="bind", + self.getMultipartData()) + dirObj = self.DirectoryObject + mdirectory = dirObj.directory + for srcDir, destDir, fileSystem, isFormat, partTable \ + in mulipartDataBind: + realDestDir = pathJoin(mdirectory, destDir) + realSrcDir = pathJoin(mdirectory, srcDir) + self._mountBind(realSrcDir, realDestDir) + isFormat = False + partObj = PartitionDistributive(realSrcDir, flagRemoveDir=False, + fileSystem=fileSystem, + isFormat=isFormat, + parent=dirObj) + DirectoryDistributive(realDestDir,parent=partObj) + + def getMultipartData(self): + """Get multipartition data""" + mulipartData = zip(self.multipartition.getPartitions(), + self.multipartition.getMountPoints(), + self.multipartition.getFileSystems(), + self.multipartition.getIsFormat(), + self.multipartition.getPartitionTable()) + return mulipartData + + def convertToDirectory(self): + """Convert partition to directory by mounting""" + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,DirectoryDistributive) and \ + mdirectory in child.directory: + return child + mdirectory = self._getMntDirectory(mdirectory) + self._mountPartition(self.partition,mdirectory) + dirObj = DirectoryDistributive(mdirectory,parent=self) + if self.multipartition: + mulipartDataNotBind = filter(lambda x: x[2]!="bind", + self.getMultipartData()) + for dev, mountPoint, fileSystem, isFormat, partTable\ + in mulipartDataNotBind: + if fileSystem!="swap": + realMountPoint = pathJoin(mdirectory, mountPoint) + self._mountPartition(dev,realMountPoint,"-t %s"%fileSystem) + partObj = PartitionDistributive(dev, flagRemoveDir=False, + fileSystem=fileSystem, + isFormat=isFormat, + parent=dirObj) + if fileSystem!="swap": + DirectoryDistributive(realMountPoint,parent=partObj) + self.DirectoryObject = dirObj + return dirObj + + def formatAllPartitions(self): + """Format all partitions""" + FS,DEV,NEEDFORMAT,NEWID,PARTTABLE = 0,1,2,3,4 + # get all information to matrix + dataPartitions = zip(self.multipartition.getFileSystems() +\ + [self.fileSystem], + self.multipartition.getPartitions() + \ + [self.partition], + self.multipartition.getIsFormat() + \ + [self.isFormat], + self.multipartition.getSystemId() + \ + [self.systemId], + self.multipartition.getPartitionTable() + \ + [self.partitionTable]) + # get partition which need format + formatPartitions = map(lambda x: (x[FS],x[DEV]), + filter(lambda x: x[NEEDFORMAT] and x[FS]!="bind", + dataPartitions)) + # format all get partition + for fileSystem, dev in formatPartitions: + if fileSystem=="swap": + self.formatSwapPartition(dev) + else: + if dev == self.partition: + self.formatPartition(dev, format=fileSystem, + label=self.rootLabel) + else: + self.formatPartition(dev, format=fileSystem) + # change system id for partitions + changeidPartitions = map(lambda x: (x[NEWID],x[DEV],x[PARTTABLE]), + filter(lambda x: x[NEWID], + dataPartitions)) + for systemid, dev, partTable in changeidPartitions: + self.changeSystemID(dev,systemid,partTable) + return True + + def _checkMount(self,dev): + """Checking mount point""" + if isMount(dev): + raise DistributiveError( + _("Failed to format partition %s, because it is mounted")%dev) + + def formatPartition(self, dev,format="reiserfs",label=""): + """Format partition""" + if not format in self.formatUtilities: + raise DistributiveError( + _("Specified format of '%s' not supported")%format) + if dev in map(lambda y: y.split(" ")[0], + filter(lambda x: x.startswith("/"), + open("/proc/swaps"))): + raise DistributiveError(\ + _("Failed to format partition %s, because it is used as swap")% + dev) + self._checkMount(dev) + if not os.access(dev,os.W_OK): + raise DistributiveError(_("Failed to format the partition") + + " %s:\n%s"%(dev,_("Permission denied"))) + if label: + labelStr = self.labelForUtilities.get(format,"") + if labelStr: + labelStr = labelStr%label + else: + labelStr = "" + execStr = self.formatUtilities[format]%(labelStr,dev) + res,errmes = self.runOsCommand(execStr) + if res == 0: + return True + else: + raise DistributiveError(_("Failed to format the partition") + + " %s:\n%s"%(dev,errmes)) + + def performFormat(self): + """Perform format for all partition of this distributive""" + if self.multipartition: + self.formatAllPartitions() + elif self.isFormat: + self.formatPartition(self.partition,format=self.fileSystem, + label=self.rootLabel) + if self.systemId: + self.changeSystemID(self.partition,self.systemId, + self.partitionTable) + + + def changeSystemID(self,dev,systemid,parttable): + """Change partition id, specified by systemid""" + deviceName = detectDeviceForPartition(dev) + if deviceName is None: + raise DistributiveError( + _("Failed to determine the parent device for %s")%dev) + # device hasn't any partition + elif deviceName == "": + return True + fdiskProg, gdiskProg = checkUtils('/sbin/fdisk','/usr/sbin/gdisk') + partitionNumber = \ + getUdevDeviceInfo(name=dev).get('ID_PART_ENTRY_NUMBER','') + devicePartitionCount = countPartitions(deviceName) + if deviceName and not partitionNumber: + raise DistributiveError( + _("Failed to determine the partition number for %s")%dev) + if parttable == "dos": + fdisk = process(fdiskProg,deviceName,stderr=STDOUT) + pipe = Popen([fdiskProg,deviceName], + stdin=PIPE, stdout=PIPE,stderr=PIPE) + if devicePartitionCount > 1: + pipe.stdin.write("t\n%s\n%s\nw\n"%(partitionNumber, + systemid)) + else: + pipe.stdin.write("t\n%s\nw\n"%systemid) + pipe.stdin.close() + pipe.wait() + elif parttable == "gpt": + pipe = Popen([gdiskProg,deviceName], + stdin=PIPE, stdout=PIPE,stderr=PIPE) + if devicePartitionCount > 1: + pipe.stdin.write("t\n%s\n%s\nw\ny\n"%(partitionNumber, + systemid)) + else: + pipe.stdin.write("t\n%s\nw\ny\n"%systemid) + pipe.stdin.close() + pipe.wait() + for waittime in (0.1,0.2,0.5,1,2,4): + if path.exists(dev): + return True + else: + sleep(waittime) + raise DistributiveError( + _("Failed to found partition %s after changing the system id")%dev) + + def formatSwapPartition(self, dev): + """Format swap partition""" + if dev in map(lambda y: y.split(" ")[0], + filter(lambda x: x.startswith("/"), + open("/proc/swaps"))): + raise DistributiveError(\ + _("Failed to execute 'mkswap %s', the swap partition is used " + "by the current system") %dev) + if isMount(dev): + raise DistributiveError( + _("Failed to format partition %s, because it is mounted") %dev) + execStr = self.formatUtilities["swap"]%dev + res,errmes = self.runOsCommand(execStr) + if res == 0: + return True + else: + raise DistributiveError(_("Failed to execute '%s'")%execStr +\ + "\n%s" %errmes) + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # get currect partition as directory + distrTo = self.convertToDirectory() + # install into directroy distributive from source + distrTo.installFrom(source) + +class ArchiveDistributive(Distributive): + def __init__(self,file,parent=None,mdirectory="/var/calculate/tmp/stage"): + Distributive.__init__(self,parent=parent) + self.file = file + self.mdirectory = mdirectory + + def _detectArchive(self,file): + """Detect archive by "/usr/bin/file" command + + Return bzip2,gzip,7z or None + """ + res,mes = self.runOsCommand("/usr/bin/file %s"%file) + if "bzip2 compressed data" in mes: + return "bzip2" + elif "gzip compressed data" in mes: + return "gzip" + elif "7-zip archive data" in mes: + return "7z" + elif file and file.endswith(".tar.lzma"): + if path.exists('/usr/bin/7za'): + return "7z" + else: + return "lzma" + return None + + def _unpackArchive(self,archfile,directory): + """Unpack archive""" + # archive is exists + if not path.exists(archfile): + raise DistributiveError(_("File '%s' not found")%archfile) + # detect type archive + archiveType = self._detectArchive(archfile) + # make directory if archive was detected normally + if archiveType: + self._makeDirectory(directory) + # unpack archive + if archiveType == "7z": + res,mes = self.runOsCommand("7za x -so %s | tar xf - -C %s/"% + (archfile,directory)) + elif archiveType == "lzma": + res,mes = self.runOsCommand("lzma -dc %s | tar xf - -C %s/"% + (archfile,directory)) + elif archiveType == "bzip2": + res,mes = self.runOsCommand("tar xjf %s -C %s/"% + (archfile,directory)) + elif archiveType == "gzip": + res,mes = self.runOsCommand("tar xf %s -C %s/"% + (archfile,directory)) + else: + raise DistributiveError(_("Unknown archive type by '%s'")% + archfile) + if res != 0: + raise DistributiveError(_("Error during unpacking\n%s")%mes) + + def unpackTo(self,directory): + """Unpack currect archive to directory""" + self._unpackArchive(self.file,directory) + + def convertToDirectory(self): + """Get archive as directory (unpack to directory)""" + # check may be the archive already unpacked + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,DirectoryDistributive) and \ + mdirectory in child.directory: + return child + # get temporary directory for unpacking + mdirectory = self._getMntDirectory(mdirectory) + dirdist = DirectoryDistributive(mdirectory,parent=self) + self._unpackArchive(self.file,mdirectory) + return dirdist + + def releaseChild(self,child): + """Remove child Directory distributive""" + if isinstance(child,DirectoryDistributive): + self._removeDirectory(child.directory) + child.directory = None + + def packToArchive(self,directory,file): + res,errmes = self.runOsCommand("tar cf %s -C %s ."%(file,directory)) + if res != 0: + raise DistributiveError(_("Failed to create the archive") + + " '%s':\n%s"%(file,errmes)) + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # get source distributive as directory distributive + dFrom = source.convertToDirectory() + # install into directroy distributive from source + self.packToArchive(dFrom.directory, self.file) + +class SquashDistributive(Distributive): + def __init__(self,file,parent=None,mdirectory="/mnt/livecd",exclude=None, + compress=""): + Distributive.__init__(self,parent=parent) + self.file = file + self.mdirectory = mdirectory + self.exclude = [] if not exclude else exclude + self.compress = compress \ + if compress and compress != "gzip" else \ + "" + + def _mountSquash(self,file,directory): + """Mount squashfs to directory""" + self._makeDirectory(directory) + self._mountToDirectory(file,directory,mountopts="-o loop -t squashfs") + + def _umountSquash(self,directory): + self._umountDirectory(directory) + self._removeDirectory(directory) + + def convertToDirectory(self): + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,DirectoryDistributive) and \ + mdirectory in child.directory: + return child + mdirectory = self._getMntDirectory(mdirectory) + self._mountSquash(self.file,mdirectory) + return DirectoryDistributive(mdirectory,parent=self) + + def releaseChild(self,child): + """Umount child Directory distributive""" + if isinstance(child,DirectoryDistributive): + self._umountSquash(child.directory) + child.directory = None + + def packToSquash(self,directory,file): + mksquashfsUtil = '/usr/bin/mksquashfs' + if not path.exists(mksquashfsUtil): + raise DistributiveError(_("Failed to create squash") + + " : %s"%_("command '%s' not found")%mksquashfsUtil) + cmd = [mksquashfsUtil, "%s/"%directory,file, "-no-progress"] + if self.exclude: + cmd += ["-e"] + self.exclude + if self.compress: + cmd += ["-comp",self.compress] + processMkSquash = process(*cmd) + if processMkSquash.failed(): + raise DistributiveError(_("Failed to create squashfs") + + " '%s':\n%s"%(file,processMkSquash.read())) + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # get source distributive as directory distributive + dFrom = source.convertToDirectory() + # install into directroy distributive from source + self.packToSquash(dFrom.directory, self.file) + +class IsoDistributive(Distributive): + def __init__(self,file,parent=None,mdirectory="/mnt/cdrom", + bdirectory="/var/calculate/tmp/iso",exclude=None, + compress="gzip"): + Distributive.__init__(self,parent=parent) + self.file = file + if path.isdir(self.file): + self.mdirectory = self.file + else: + self.mdirectory = mdirectory + if file == bdirectory: + self.bdirectory = file + else: + self.bdirectory = self._getMntDirectory(bdirectory) + self.exclude = [] if not exclude else exclude + self.compress = compress + + def probe(self): + """Check directory for iso content""" + try: + pathname = self.getIsoContentDirectory() + except: + return False + return path.exists(path.join(pathname,"syslinux")) and \ + path.exists(path.join(pathname,"isolinux")) + + def _mountIso(self,file,directory): + if self.file != self.mdirectory: + self._makeDirectory(directory) + tf = typeFile(magic=0x6) + ftype = tf.getMType(file) + if "block special" in ftype: + mopts = "" + else: + mopts = "-o loop" + self._mountToDirectory(file,directory,mountopts=mopts) + + def _umountIso(self,directory): + if self.file != self.mdirectory: + self._umountDirectory(directory) + self._removeDirectory(directory) + + def convertToSquash(self): + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,SquashDistributive) and \ + mdirectory in child.file: + return child + if self.mdirectory != self.file: + mdirectory = self._getMntDirectory(mdirectory) + self._mountIso(self.file,mdirectory) + fileLive = self._getLastLive(mdirectory) + if not fileLive: + self._umountIso(mdirectory) + raise DistributiveError(_("Iso %s contains no live image") % + self.file) + return SquashDistributive(path.join(mdirectory,fileLive), + parent=self,exclude=self.exclude, + compress=self.compress) + + def getIsoContentDirectory(self): + """Return directory with content of iso image""" + squash = self.convertToSquash() + return path.dirname(squash.file) + + def releaseChild(self,child): + """Umount child Directory distributive""" + if isinstance(child,SquashDistributive): + self._umountIso(path.dirname(child.file)) + child.directory = None + + def convertToDirectory(self): + return self.convertToSquash().convertToDirectory() + + def packToIso(self,directory,file): + # remove previous version of iso + try: + if path.exists(file): + os.unlink(file) + except Exception, e: + raise DistributiveError(_("Failed to remove") +\ + " %s:\n%s"%(file,str(e))) + except KeyboardInterrupt, e: + self.setSignalInterrupt() + raise DistributiveError(_("Failed to remove") +\ + " %s:\n%s"%(file,str(e))) + + mkisoUtil = '/usr/bin/mkisofs' + if not path.exists(mkisoUtil): + raise DistributiveError(_("Failed to create the iso image") + + " : %s"%_("command '%s' not found")%mkisoUtil) + res,errmes = self.runOsCommand( + "%(progname)s %(params)s -o %(target)s %(source)s/"% + {'progname':mkisoUtil, + 'params':" ".join(["-b isolinux/isolinux.bin","-no-emul-boot", + "-boot-load-size 4","-boot-info-table","-iso-level 4", + "-hide boot.catalog"]), + 'target':file, + 'source':directory}) + if res == 0: + return True + else: + raise DistributiveError(_("Failed to create the iso image") + + " %s:\n%s"%(file,errmes)) + + def prepareIso(self): + raise DistributiveError( + _("Iso image cannot be created without overriding prepareIso")) + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # make temporary directory for creating iso image + isoDirectory = self.bdirectory + self._makeDirectory(isoDirectory) + + try: + # getting squash from source + liveimage = self._getLastLive(isoDirectory) + if liveimage: + curnum = self._getSquashNum(self.reLive.search(liveimage)) + liveimage = "livecd.squashfs.%d"%(curnum+1) + else: + liveimage = "livecd.squashfs" + if isinstance(source,SquashDistributive): + self._copyfile(source.file, + path.join(isoDirectory,liveimage)) + else: + distDirectory = source.convertToDirectory() + squashDistr = SquashDistributive( + path.join(isoDirectory,liveimage), + exclude=self.exclude, + compress=self.compress) + squashDistr.installFrom(distDirectory) + # prepare iso + self.prepareIso(isoDirectory) + # pack iso + if self.bdirectory != self.file: + self.packToIso(isoDirectory, self.file) + except DistributiveError,e: + raise e + except KeyboardInterrupt,e: + self.setSignalInterrupt() + raise DistributiveError(_("Keyboard interruption")) + + def close(self): + # close all child + Distributive.close(self) + # remove temporary directory + if path.lexists(self.bdirectory) and self.file != self.bdirectory: + self._removeDirectory(self.bdirectory) + +class FlashDistributive(PartitionDistributive): + def _checkMount(self,dev): + """Checking mount point""" + mp = isMount(dev) + if mp: + if mp.startswith('/media'): + self._umountDirectory(mp) + else: + raise DistributiveError( + _("Failed to format partition %s, because it is mounted")%dev) + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # make temporary directory for creating iso image + distrTo = self.convertToDirectory() + # getting squash from source + if isinstance(source,IsoDistributive): + self.rsync(source.getIsoContentDirectory(),distrTo.directory) + else: + raise DistributiveError( + _("Installation to flash does not support %s"% + source.__class__.__name__)) + +class PxeDistributive(Distributive): + def __init__(self,directory,parent=None): + Distributive.__init__(self,parent=parent) + self.directory = path.join(directory,"calculate") + self.origdir = directory + + def getDirectory(self): + return self.origdir + + def installFrom(self,source): + # make temporary directory for creating iso image + distrTo = self.directory + # getting squash from source + if isinstance(source,IsoDistributive): + if path.exists(self.directory): + removeDir(self.directory) + self._makeDirectory(self.directory) + self.rsync(source.getIsoContentDirectory(),distrTo) + else: + raise DistributiveError( + _("Installation for PXE does not support %s"% + source.__class__.__name__)) + +class ScratchDistributive(Distributive): + def __init__(self,directory,parent=None,mdirectory="/mnt/install", + check=False): + Distributive.__init__(self,parent=parent) + self.directory = directory + self.mdirectory = mdirectory + if check and not (path.exists(path.join(directory,"workspace")) and \ + path.exists(path.join(directory,"delta")) and \ + path.exists(path.join(directory,"calculate"))): + raise DistributiveError( + _("Wrong scratch distribution in '%s'")%directory) + + def _mountLayers(self,source,target): + """Mount squashfs to directory""" + self._makeDirectory(target) + self._mountToDirectory("none",target, + mountopts="-t aufs "+\ + "-o udba=reval,br:%(work)s=rw:%(delta)s=ro+wh:%(static)s=ro" %\ + {"work":path.join(source,"workspace"), + "delta":path.join(source,"delta"), + "static":path.join(source,"calculate")}) + + def _umountLayers(self,directory): + self._umountDirectory(directory) + self._removeDirectory(directory) + + def _mountLiveImage(self,file,directory): + """Mount squashfs to directory""" + self._makeDirectory(directory) + self._mountToDirectory(file,directory,mountopts="-o loop -t squashfs") + + def _umountLiveImage(self,directory): + self._umountDirectory(directory) + + def convertToDirectory(self): + """Convert scrach directories to one directory""" + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,DirectoryDistributive) and \ + mdirectory in child.directory: + return child + mdirectory = self._getMntDirectory(mdirectory) + liveFile = self._getLastLive(self.directory) + self._mountLiveImage(path.join(self.directory,liveFile), + path.join(self.directory,"calculate")) + self._mountLayers(self.directory,mdirectory) + return DirectoryDistributive(mdirectory,parent=self) + + def releaseChild(self,child): + """Umount child Directory distributive""" + if isinstance(child,DirectoryDistributive): + self._umountLayers(child.directory) + self._umountLiveImage(path.join(self.directory,"calculate")) + child.directory = None + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # get source distributive as directory distributive + dFrom = source.convertToDirectory() + # install into directroy distributive from source + self.packToSquash(dFrom.directory, self.file) + +class ScratchPartitionDistributive(PartitionDistributive): + + def releaseChild(self,child): + """Umount child Directory distributive""" + if isinstance(child,ScratchDistributive): + self._umountPartition(child.directory) + child.directory = None + + def convertToDirectory(self): + """Convert scratch partition to directory by scratch directory""" + return self.convertToScratch().convertToDirectory() + + def convertToScratch(self): + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,ScratchDistributive) and \ + mdirectory in child.directory: + return child + mdirectory = self._getMntDirectory(mdirectory) + self._mountPartition(self.partition,mdirectory) + return ScratchDistributive(mdirectory,parent=self) + + def prepareScratch(self,directory): + """Create need scratch directories""" + for scrDirectory in ("calculate","delta","workspace"): + self._makeDirectory(path.join(directory,scrDirectory)) + + def getBootDirectory(self): + """Get directory which contains boot""" + return path.join(self.convertToScratch().directory,"boot") + + def postinstallMountBind(self): + pass + + def installFrom(self, source): + """Install distributive to partition from source distributive""" + # get currect partition as directory + distrTo = self.convertToScratch() + scratchDirectory = distrTo.directory + + try: + # getting squash from source + if isinstance(source,IsoDistributive): + source = source.convertToSquash() + if isinstance(source,SquashDistributive): + self._copyfile(source.file, + path.join(scratchDirectory,"livecd.squashfs")) + else: + distDirectory = source.convertToDirectory() + squashDistr = SquashDistributive( + path.join(scratchDirectory,"livecd.squashfs")) + squashDistr.installFrom(distDirectory) + + # prepare scratch + self.prepareScratch(scratchDirectory) + except DistributiveError,e: + raise e + except KeyboardInterrupt,e: + self.setSignalInterrupt() + raise DistributiveError(_("Keyboard interruption"))