diff --git a/install/distr.py b/install/distr.py index e6202e8..8587509 100644 --- a/install/distr.py +++ b/install/distr.py @@ -27,6 +27,7 @@ from subprocess import Popen,PIPE,STDOUT from contextlib import nested from itertools import * from functools import partial +from calculate.lib.datavars import VariableError from calculate.lib.utils.files import (runOsCommand,isMount,removeDir, processProgress,countFiles,STDOUT, @@ -88,12 +89,25 @@ def progressCopyFile(source,dest): outfile.write(infile.read(bufsize)) yield i + class DistributiveError(Exception): """Error for distributive operations""" pass + class Distributive(object): - """Distributive object. Parent object for all distributive.""" + """ + Distributive object. Parent object for all distributive. + """ + + class Type: + Directory = "dir" + Partition = "part" + SquashFS = "squash" + Archive = "arch" + Iso = "iso" + Layered = "layered" + mountError = _("Failed to mount") + " %s:\n%s" reLive = re.compile(r"^live[^.]*\.squashfs(\.(\d+))?$",re.S) contentCache = {} @@ -102,6 +116,7 @@ class Distributive(object): def __init__(self, parent=None): self.childs = [] self.ref = 0 + self.locked = False # if specified parent if parent: # save parent type for correct resource release @@ -115,11 +130,11 @@ class Distributive(object): def fromFile(cls,filename): """Get Distributive object by filename""" # MAGIC_COMPRESS 0x000004 Check inside compressed files - tf = typeFile(magic=MAGIC_COMPRESS|MAGIC_SYMLINK|MAGIC_CONTINUE) + tf = typeFile(magic=MAGIC_COMPRESS | MAGIC_SYMLINK | MAGIC_CONTINUE) ftype = tf.getMType(filename) if ftype: if "block special" in ftype: - for distrType in (IsoDistributive,FlashDistributive, + for distrType in (IsoDistributive, FlashDistributive, PartitionDistributive): distr = distrType(filename) res = distr.probe() @@ -129,17 +144,20 @@ class Distributive(object): if "ISO 9660 CD-ROM" in ftype: return IsoDistributive(filename) elif "7-zip" in ftype or \ - "POSIX tar archive" in ftype: + "POSIX tar archive" in ftype: return ArchiveDistributive(filename) elif "Squashfs filesystem" in ftype: return SquashDistributive(filename) elif path.isdir(filename): - if path.isfile(path.join(filename,"livecd")): + if path.isfile(path.join(filename, "livecd")): return IsoDistributive(filename) else: - return DirectoryDistributive(filename) - raise DistributiveError(_("Wrong distribution") + " '%s':\n%s"\ - %(filename,ftype)) + important_dirs = ("etc", "lib", "bin", "sbin", "var") + if all(path.exists(path.join(filename, x)) + for x in important_dirs): + return DirectoryDistributive(filename) + raise DistributiveError(_("Wrong distribution") + " '%s':\n%s" \ + % (filename, ftype)) def getType(self): return _("empty") @@ -160,11 +178,17 @@ class Distributive(object): """ self.parent = None + def reserve(self): + self.locked = True + + def release(self): + self.locked = False + def __enter__(self): self.ref += 1 return self - def __exit__(self,type,value,traceback): + def __exit__(self, type, value, traceback): self.ref -= 1 if not self.ref: self.close() @@ -180,21 +204,27 @@ class Distributive(object): dist2 = dist1.convertToDirectory() dist1.close() """ + #print "Close", self + if self.locked: + return False if self.ref: self.ref -= 1 return False else: - # close all child - if self.childs: - for child in reversed(self.childs): - # check detach - if child.parent: - child.close() - self.childs = [] - # if has parent - if self.parent: - self.parent.releaseChild(self) + try: + # close all child + if self.childs: + for child in reversed(self.childs): + # check detach + if child.parent: + child.close() + self.childs = [] + # if has parent + if self.parent: + self.parent.releaseChild(self) + finally: self.parent = None + self.childs = [] return True def releaseChild(self,child): @@ -410,23 +440,28 @@ class Distributive(object): newDirectoryName = "%s.%s"%(directory,self.rndString()) return newDirectoryName - def getInfoFromDirectory(self,directory): + @staticmethod + def getInfoFromDirectory(directory): d = {} - if path.lexists(path.join(directory,'lib64')): - d['os_arch_machine'] = 'x86_64' - elif path.lexists(path.join(directory,'lib')): - d['os_arch_machine']= 'i686' - else: - d['os_arch_machine'] = '' - dv = LinuxDataVars(systemRoot=directory) - d["os_linux_shortname"] = dv.Get('os_linux_shortname') - d['os_linux_ver'] = dv.Get('os_linux_ver') - d['os_linux_build'] = dv.Get('os_linux_build') - d['os_linux_name'] = dv.Get('os_linux_name') - d['os_linux_subname'] = dv.Get('os_linux_subname') - d['os_linux_system'] = dv.Get('os_linux_system') - # make lazy call - d['os_linux_files'] = partial(dv.Get,'os_linux_files') + try: + if path.lexists(path.join(directory,'lib64')): + d['os_arch_machine'] = 'x86_64' + elif path.lexists(path.join(directory,'lib')): + d['os_arch_machine']= 'i686' + else: + d['os_arch_machine'] = '' + dv = LinuxDataVars(systemRoot=directory) + d["os_linux_shortname"] = dv.Get('os_linux_shortname') + d['os_linux_ver'] = dv.Get('os_linux_ver') + d['os_linux_build'] = dv.Get('os_linux_build') + d['os_linux_name'] = dv.Get('os_linux_name') + d['os_linux_subname'] = dv.Get('os_linux_subname') + d['os_linux_system'] = dv.Get('os_linux_system') + d['cl_make_profile'] = dv.Get('cl_make_profile') + # make lazy call + d['os_linux_files'] = partial(dv.Get, 'os_linux_files') + except VariableError: + pass return d.copy() def getInfo(self,filename=None): @@ -436,12 +471,9 @@ class Distributive(object): # get from cache keyCache = None if not filename: - if hasattr(self,"file"): - keyCache = self.file - elif hasattr(self,"partition"): - keyCache = self.partition - elif hasattr(self,"directory"): - keyCache = self.directory + for keyname in ("file", "partition", "directory"): + if hasattr(self, keyname): + keyCache = getattr(self, keyname) elif filename in self.contentCache: keyCache = filename @@ -453,11 +485,11 @@ class Distributive(object): extname = "isodir" try: distr = self.fromFile(filename) if filename else self - mapExtName = {DirectoryDistributive:"dir", - IsoDistributive:"isodir", - FlashDistributive:"flash", - PartitionDistributive:"partdir"} - extname = mapExtName.get(distr.__class__,"") + mapExtName = {DirectoryDistributive: "dir", + IsoDistributive: "isodir", + FlashDistributive: "flash", + PartitionDistributive: "partdir"} + extname = mapExtName.get(distr.__class__, "") image = distr.convertToDirectory() except Exception as e: if distr: @@ -474,14 +506,95 @@ class Distributive(object): if image: image.close() + @classmethod + def unserialize(cls, data, parent=None): + class_mapper = {cls.Type.Directory : DirectoryDistributive, + cls.Type.Iso : IsoDistributive, + cls.Type.Partition: PartitionDistributive, + cls.Type.SquashFS: SquashDistributive, + cls.Type.Layered: LayeredDistributive} + if not data.get('type', '') in class_mapper: + raise DistributiveError(_("Failed to unserialize type %s") % + data.get('type', '')) + return class_mapper[data['type']].unserialize(data, parent=parent) + + @staticmethod + def required(*params): + def decor(f): + def wrapper(cls, data, **kw): + if any(not x in data for x in params): + raise DirectoryDistributive( + _("Failed to unserialize %s") % cls.__class__.__name__) + return f(cls,data, **kw) + + return wrapper + + return decor + + def restore(self): + raise DistributiveError(_("Recovery is not implemented")) + + def is_invalid(self): + try: + return not "cl_make_profile" in self.getInfo() + except DistributiveError: + return True + class DirectoryDistributive(Distributive): + """ + Дистрибутив в директории + """ + data = [{'name': 'proc', + 'type': 'proc', + 'target': 'proc', + 'source': 'none'}, + {'name': 'sys', + 'type': 'sysfs', + 'target': 'sys', + 'source': 'none'}, + {'name': 'dev', + 'type': 'bind', + 'target': 'dev', + 'source': '/dev'}, + {'name': 'devpts', + 'type': 'bind', + 'target': 'dev/pts', + 'source': '/dev/pts'}, + {'name': 'remote', + 'type': 'bind', + 'target': 'var/calculate/remote', + 'source': '/var/calculate/remote'}, + ] + def __init__(self,directory,parent=None,mdirectory=None): Distributive.__init__(self,parent=parent) self.directory = directory self.mdirectory = mdirectory + self.system_mounted = False if not parent: self._makeDirectory(self.directory) + def mountSystemDirectories(self, skip=("remote",)): + """ + Подключить к дистрибутиву системые ресурсы (/proc, /sys) + :return: + """ + for obj in filter(lambda x: x['name'] not in skip, self.data): + target_path = path.join(self.directory, obj['target']) + if obj['type'] == 'bind': + self._mountToBind(obj['source'], target_path) + else: + self._mountToDirectory(obj['source'],target_path, + "-t %s"%obj['type']) + self.system_mounted = True + + def umountSystemDirectories(self): + for obj in reversed(self.data): + target_path = path.join(self.directory, obj['target']) + if isMount(target_path): + self._umountDirectory(target_path) + self.system_mounted = False + def getType(self): return _("directory '%s'")%self.directory @@ -502,6 +615,11 @@ class DirectoryDistributive(Distributive): self._removeDirectory(child.directory) child.directory = None + def close(self): + if self.system_mounted: + self.umountSystemDirectories() + Distributive.close(self) + def convertToDirectory(self): if self.mdirectory: return self.bindDirectory(self.mdirectory) @@ -513,11 +631,11 @@ class DirectoryDistributive(Distributive): execStr = '/bin/rm -rf --one-file-system %s'%self.directory res,errmes = self.runOsCommand(execStr) if res == 0: + self._makeDirectory(self.directory) return True else: - raise DistributiveError(_("Failed to format the partition") + - " %s:\n%s"%(dev,errmes)) - self._makeDirectory(self.directory) + raise DistributiveError(_("Failed to clean directory") + + " %s:\n%s" % (self.directory, errmes)) def installFrom(self, source, **kwargs): """Install distributive to directory from source distributive""" @@ -529,6 +647,27 @@ class DirectoryDistributive(Distributive): # copy distributive from source to this self.rsync(dFrom.directory,self.directory,**kwargs) + def serialize(self): + d = {'type': Distributive.Type.Directory, + 'directory': self.directory, + 'system_mounted':self.system_mounted, + 'childs': [x.serialize() for x in self.childs]} + if self.mdirectory: + d['mdirectory'] = self.mdirectory + return d + + @classmethod + @Distributive.required("directory", "system_mounted") + def unserialize(self, data, parent=None): + ld = DirectoryDistributive(data['directory'],parent=parent) + ld.system_mounted = data['system_mounted'] + if "mdirectory" in data: + ld.mdirectory = data['mdirectory'] + ld.childs = [Distributive.unserialize(x, parent=ld) for x in + data.get('childs', [])] + return ld + + class DataPartition: """Data partition""" dev = None @@ -600,19 +739,19 @@ class MultiPartitions: class PartitionDistributive(Distributive): - formatUtilities = { 'ext2':'/sbin/mkfs.ext2 %s %s', - 'ext3':'/sbin/mkfs.ext3 %s %s', - 'ext4':'/sbin/mkfs.ext4 %s %s', - 'jfs':'/sbin/mkfs.jfs %s -f %s', - 'reiserfs':'/sbin/mkfs.reiserfs %s -f %s', - 'btrfs':'/sbin/mkfs.btrfs -f %s %s', - 'nilfs2':'/sbin/mkfs.nilfs2 %s %s', - 'xfs':'/sbin/mkfs.xfs %s -f %s', - 'vfat':'/usr/sbin/mkfs.vfat %s -F 32 %s', - 'ntfs-3g':'/usr/sbin/mkfs.ntfs %s -FQ %s', - 'ntfs':'/usr/sbin/mkfs.ntfs %s -FQ %s', - 'uefi':'/usr/sbin/mkfs.vfat %s %s', - 'swap':'/sbin/mkswap %s' + formatUtilities = {'ext2':'/sbin/mkfs.ext2 %s %s', + 'ext3':'/sbin/mkfs.ext3 %s %s', + 'ext4':'/sbin/mkfs.ext4 %s %s', + 'jfs':'/sbin/mkfs.jfs %s -f %s', + 'reiserfs':'/sbin/mkfs.reiserfs %s -f %s', + 'btrfs':'/sbin/mkfs.btrfs -f %s %s', + 'nilfs2':'/sbin/mkfs.nilfs2 %s %s', + 'xfs':'/sbin/mkfs.xfs %s -f %s', + 'vfat':'/usr/sbin/mkfs.vfat %s -F 32 %s', + 'ntfs-3g':'/usr/sbin/mkfs.ntfs %s -FQ %s', + 'ntfs':'/usr/sbin/mkfs.ntfs %s -FQ %s', + 'uefi':'/usr/sbin/mkfs.vfat %s %s', + 'swap':'/sbin/mkswap %s' } labelForUtilities = { 'ext2':'-L %s', 'ext3':'-L %s', @@ -922,6 +1061,25 @@ class PartitionDistributive(Distributive): # install into directroy distributive from source distrTo.installFrom(source,**kwargs) + def serialize(self): + d = {'type': Distributive.Type.Partition, + 'partition': self.partition, + 'mdirectory': self.mdirectory, + 'flag_remove_directory': self.flagRemoveDir, + 'childs': [x.serialize() for x in self.childs]} + return d + + @classmethod + @Distributive.required("partition", "mdirectory", "flag_remove_directory") + def unserialize(self, data, parent=None): + ld = PartitionDistributive(data['directory'], parent=parent) + ld.mdirectory = data['mdirectory'] + ld.flagRemoveDir = data['flag_remove_directory'] + ld.childs = [Distributive.unserialize(x, parent=ld) for x in + data.get('childs', [])] + return ld + + class ArchiveDistributive(Distributive): def __init__(self,file,parent=None,mdirectory="/var/calculate/tmp/stage"): Distributive.__init__(self,parent=parent) @@ -1016,6 +1174,22 @@ class ArchiveDistributive(Distributive): # install into directroy distributive from source self.packToArchive(dFrom.directory, self.file) + def serialize(self): + d = {'type': Distributive.Type.Archive, + 'file': self.file, + 'mdirectory':self.mdirectory, + 'childs': [x.serialize() for x in self.childs]} + return d + + @classmethod + @Distributive.required("mdirectory","file") + def unserialize(self, data, parent=None): + ld = ArchiveDistributive(data['file'], + mdirectory=data['mdirectory'], parent=parent) + ld.childs = [Distributive.unserialize(x, parent=ld) + for x in data.get('childs',[])] + return ld + class SquashDistributive(Distributive): def __init__(self,file,parent=None,mdirectory="/mnt/livecd",exclude=None, compress=""): @@ -1078,6 +1252,23 @@ class SquashDistributive(Distributive): # install into directroy distributive from source self.packToSquash(dFrom.directory, self.file) + def serialize(self): + d = {'type': Distributive.Type.SquashFS, + 'file': self.file, + 'mdirectory':self.mdirectory, + 'childs': [x.serialize() for x in self.childs]} + return d + + @classmethod + @Distributive.required("mdirectory","file") + def unserialize(self, data, parent=None): + ld = SquashDistributive(data['file'], + mdirectory=data['mdirectory'], parent=parent) + ld.childs = [Distributive.unserialize(x, parent=ld) + for x in data.get('childs',[])] + return ld + + class IsoDistributive(Distributive): def __init__(self,file,parent=None,mdirectory="/mnt/cdrom", bdirectory="/var/calculate/tmp/iso",exclude=None, @@ -1237,6 +1428,25 @@ class IsoDistributive(Distributive): if path.lexists(self.bdirectory) and self.file != self.bdirectory: self._removeDirectory(self.bdirectory) + def serialize(self): + d = {'type': Distributive.Type.Iso, + 'file': self.file, + 'mdirectory':self.mdirectory, + 'bdirectory':self.bdirectory, + 'childs': [x.serialize() for x in self.childs]} + return d + + @classmethod + @Distributive.required("mdirectory","bdirectory","file") + def unserialize(self, data, parent=None): + ld = IsoDistributive(data['file'], + mdirectory=data['mdirectory'], + bdirectory=data['bdirectory'], + parent=parent) + ld.childs = [Distributive.unserialize(x, parent=ld) + for x in data.get('childs',[])] + return ld + class FlashDistributive(PartitionDistributive): def _checkMount(self,dev): """Checking mount point""" @@ -1450,3 +1660,105 @@ class PxeDistributive(Distributive): raise DistributiveError( _("PXE install does not support %s"% source.__class__.__name__)) + + +class LayeredDistributive(Distributive): + """ + Каталог дистрибутива для сборки + """ + needFormat = False + + def __init__(self, mdirectory, diff_directory, image_file=None, + parent=None): + """ + :param mdirectory: результирующий каталог + :param diff_directory: каталог содержит изменения от оригинала + :param image_file: образ оригинала + :param parent: родительский дистрибутив + """ + super(LayeredDistributive, self).__init__(parent=parent) + self.mdirectory = mdirectory + self.diff_directory = diff_directory + self.image_mount_dir = None + self.image_distro = None + self.image_file = image_file + + def getType(self): + return _("layered '%s %s'")%(self.image_file, self.diff_directory) + + def clearDiff(self): + if path.exists(self.diff_directory): + self._removeDirectory(self.diff_directory) + + def _mountLayers(self, target): + """Mount squashfs to directory""" + self._makeDirectory(target) + if not path.exists(self.diff_directory): + self._makeDirectory(self.diff_directory) + self._mountToDirectory("none", target, mountopts=( + "-t aufs -o udba=reval,br:%(work)s=rw:%(static)s=ro" % + {"work": self.diff_directory, + "static": self.image_mount_dir})) + + def _umountLayers(self, directory): + self._umountDirectory(directory) + self._removeDirectory(directory) + + def _mountLiveImage(self): + """Mount squashfs to directory""" + self.image_distro = IsoDistributive(file=self.image_file, parent=self) + self.image_mount_dir = ( + self.image_distro.convertToDirectory().getDirectory()) + + def _umountLiveImage(self): + if self.image_distro: + self.image_distro.close() + self.image_distro = None + + def convertToDirectory(self): + """Convert scrach directories to one directory""" + mdirectory = self.mdirectory + for child in self.childs: + if isinstance(child,DirectoryDistributive) and \ + mdirectory in child.directory: + return child + mdirectory = self._getMntDirectory(mdirectory) + self._mountLiveImage() + self._mountLayers(mdirectory) + return DirectoryDistributive(mdirectory, parent=self) + + def releaseChild(self, child): + """Umount child Directory distributive""" + if isinstance(child,DirectoryDistributive): + self._umountLayers(child.directory) + self._umountLiveImage() + child.directory = None + + def installFrom(self, source, **kwargs): + """Install distributive to partition from source distributive""" + # make temporary directory for creating iso image + if isinstance(source, (IsoDistributive, SquashDistributive)): + self.image_file = source.file + return True + raise Distributive("Not implementation") + + def serialize(self): + d = {'type': Distributive.Type.Layered, + 'mdirectory': self.mdirectory, + 'diff_directory': self.diff_directory, + 'image_file': self.image_file, + 'childs': [x.serialize() for x in self.childs]} + if self.image_mount_dir: + d['image_mount_dir'] = self.image_mount_dir + return d + + @classmethod + @Distributive.required("mdirectory", "diff_directory", "image_file") + def unserialize(self, data, parent=None): + ld = LayeredDistributive(data['mdirectory'], + data['diff_directory'], + data['image_file'], + parent=parent) + ld.childs = [Distributive.unserialize(x, parent=ld) + for x in data.get('childs',[])] + return ld diff --git a/install/variables/distr.py b/install/variables/distr.py index 001b5cd..c564c66 100644 --- a/install/variables/distr.py +++ b/install/variables/distr.py @@ -249,7 +249,7 @@ class VariableClImage(ReadonlyVariable): def get(self): """Get image file from distributive repository""" try: - if self.Get('cl_action') != 'system': + if not self.Get('cl_action') in ('system','prepare'): return Distributive.fromFile('/') filename = self.Get('cl_image_filename') if filename: