Изменения для cl-assemble

install/distr.py

@@ -27,6 +27,7 @@ from subprocess import Popen,PIPE,STDOUT
 from contextlib import nested
 from itertools import *
 from functools import partial
+from calculate.lib.datavars import VariableError
 
 from calculate.lib.utils.files import (runOsCommand,isMount,removeDir,
                     processProgress,countFiles,STDOUT,
@@ -88,12 +89,25 @@ def progressCopyFile(source,dest):
             outfile.write(infile.read(bufsize))
             yield i
 
+
 class DistributiveError(Exception):
     """Error for distributive operations"""
     pass
 
+
 class Distributive(object):
-    """Distributive object. Parent object for all distributive."""
+    """
+    Distributive object. Parent object for all distributive.
+    """
+
+    class Type:
+        Directory = "dir"
+        Partition = "part"
+        SquashFS = "squash"
+        Archive = "arch"
+        Iso = "iso"
+        Layered = "layered"
+
     mountError = _("Failed to mount") + " %s:\n%s"
     reLive = re.compile(r"^live[^.]*\.squashfs(\.(\d+))?$",re.S)
     contentCache = {}
@@ -102,6 +116,7 @@ class Distributive(object):
     def __init__(self, parent=None):
         self.childs = []
         self.ref = 0
+        self.locked = False
         # if specified parent
         if parent:
             # save parent type for correct resource release
@@ -115,11 +130,11 @@ class Distributive(object):
     def fromFile(cls,filename):
         """Get Distributive object by filename"""
         # MAGIC_COMPRESS 0x000004 Check inside compressed files 
-        tf = typeFile(magic=MAGIC_COMPRESS|MAGIC_SYMLINK|MAGIC_CONTINUE)
+        tf = typeFile(magic=MAGIC_COMPRESS | MAGIC_SYMLINK | MAGIC_CONTINUE)
         ftype = tf.getMType(filename)
         if ftype:
             if "block special" in ftype:
-                for distrType in (IsoDistributive,FlashDistributive,
+                for distrType in (IsoDistributive, FlashDistributive,
                                   PartitionDistributive):
                     distr = distrType(filename)
                     res = distr.probe()
@@ -129,17 +144,20 @@ class Distributive(object):
             if "ISO 9660 CD-ROM" in ftype:
                 return IsoDistributive(filename)
             elif "7-zip" in ftype or \
-                "POSIX tar archive" in ftype:
+                            "POSIX tar archive" in ftype:
                 return ArchiveDistributive(filename)
             elif "Squashfs filesystem" in ftype:
                 return SquashDistributive(filename)
             elif path.isdir(filename):
-                if path.isfile(path.join(filename,"livecd")):
+                if path.isfile(path.join(filename, "livecd")):
                     return IsoDistributive(filename)
                 else:
-                    return DirectoryDistributive(filename)
-        raise DistributiveError(_("Wrong distribution") + " '%s':\n%s"\
-                                    %(filename,ftype))
+                    important_dirs = ("etc", "lib", "bin", "sbin", "var")
+                    if all(path.exists(path.join(filename, x))
+                           for x in important_dirs):
+                        return DirectoryDistributive(filename)
+        raise DistributiveError(_("Wrong distribution") + " '%s':\n%s" \
+                                % (filename, ftype))
 
     def getType(self):
         return _("empty")
@@ -160,11 +178,17 @@ class Distributive(object):
         """
         self.parent = None
 
+    def reserve(self):
+        self.locked = True
+
+    def release(self):
+        self.locked = False
+
     def __enter__(self):
         self.ref += 1
         return self
 
-    def __exit__(self,type,value,traceback):
+    def __exit__(self, type, value, traceback):
         self.ref -= 1
         if not self.ref:
             self.close()
@@ -180,21 +204,27 @@ class Distributive(object):
         dist2 = dist1.convertToDirectory()
         dist1.close()
         """
+        #print "Close", self
+        if self.locked:
+            return False
         if self.ref:
             self.ref -= 1
             return False
         else:
-            # close all child
-            if self.childs:
-                for child in reversed(self.childs):
-                    # check detach
-                    if child.parent:
-                        child.close()
-                self.childs = []
-            # if has parent
-            if self.parent:
-                self.parent.releaseChild(self)
+            try:
+                # close all child
+                if self.childs:
+                    for child in reversed(self.childs):
+                        # check detach
+                        if child.parent:
+                            child.close()
+                    self.childs = []
+                # if has parent
+                if self.parent:
+                    self.parent.releaseChild(self)
+            finally:
                 self.parent = None
+                self.childs = []
         return True
 
     def releaseChild(self,child):
@@ -410,23 +440,28 @@ class Distributive(object):
             newDirectoryName = "%s.%s"%(directory,self.rndString())
         return newDirectoryName
 
-    def getInfoFromDirectory(self,directory):
+    @staticmethod
+    def getInfoFromDirectory(directory):
         d = {}
-        if path.lexists(path.join(directory,'lib64')):
-            d['os_arch_machine'] = 'x86_64'
-        elif path.lexists(path.join(directory,'lib')):
-            d['os_arch_machine']= 'i686'
-        else:
-            d['os_arch_machine'] = ''
-        dv = LinuxDataVars(systemRoot=directory)
-        d["os_linux_shortname"] = dv.Get('os_linux_shortname')
-        d['os_linux_ver'] = dv.Get('os_linux_ver')
-        d['os_linux_build'] = dv.Get('os_linux_build')
-        d['os_linux_name'] = dv.Get('os_linux_name')
-        d['os_linux_subname'] = dv.Get('os_linux_subname')
-        d['os_linux_system'] = dv.Get('os_linux_system')
-        # make lazy call
-        d['os_linux_files'] = partial(dv.Get,'os_linux_files')
+        try:
+            if path.lexists(path.join(directory,'lib64')):
+                d['os_arch_machine'] = 'x86_64'
+            elif path.lexists(path.join(directory,'lib')):
+                d['os_arch_machine']= 'i686'
+            else:
+                d['os_arch_machine'] = ''
+            dv = LinuxDataVars(systemRoot=directory)
+            d["os_linux_shortname"] = dv.Get('os_linux_shortname')
+            d['os_linux_ver'] = dv.Get('os_linux_ver')
+            d['os_linux_build'] = dv.Get('os_linux_build')
+            d['os_linux_name'] = dv.Get('os_linux_name')
+            d['os_linux_subname'] = dv.Get('os_linux_subname')
+            d['os_linux_system'] = dv.Get('os_linux_system')
+            d['cl_make_profile'] = dv.Get('cl_make_profile')
+            # make lazy call
+            d['os_linux_files'] = partial(dv.Get, 'os_linux_files')
+        except VariableError:
+            pass
         return d.copy()
 
     def getInfo(self,filename=None):
@@ -436,12 +471,9 @@ class Distributive(object):
             # get from cache
             keyCache = None
             if not filename:
-                if hasattr(self,"file"):
-                    keyCache = self.file
-                elif hasattr(self,"partition"):
-                    keyCache = self.partition
-                elif hasattr(self,"directory"):
-                    keyCache = self.directory
+                for keyname in ("file", "partition", "directory"):
+                    if hasattr(self, keyname):
+                        keyCache = getattr(self, keyname)
             elif filename in self.contentCache:
                 keyCache = filename
 
@@ -453,11 +485,11 @@ class Distributive(object):
             extname = "isodir"
             try:
                 distr = self.fromFile(filename) if filename else self
-                mapExtName = {DirectoryDistributive:"dir",
-                              IsoDistributive:"isodir",
-                              FlashDistributive:"flash",
-                              PartitionDistributive:"partdir"}
-                extname = mapExtName.get(distr.__class__,"")
+                mapExtName = {DirectoryDistributive: "dir",
+                              IsoDistributive: "isodir",
+                              FlashDistributive: "flash",
+                              PartitionDistributive: "partdir"}
+                extname = mapExtName.get(distr.__class__, "")
                 image = distr.convertToDirectory()
             except Exception as e:
                 if distr:
@@ -474,14 +506,95 @@ class Distributive(object):
             if image:
                 image.close()
 
+    @classmethod
+    def unserialize(cls, data, parent=None):
+        class_mapper = {cls.Type.Directory : DirectoryDistributive,
+                        cls.Type.Iso : IsoDistributive,
+                        cls.Type.Partition: PartitionDistributive,
+                        cls.Type.SquashFS: SquashDistributive,
+                        cls.Type.Layered: LayeredDistributive}
+        if not data.get('type', '') in class_mapper:
+            raise DistributiveError(_("Failed to unserialize type %s") %
+                                    data.get('type', ''))
+        return class_mapper[data['type']].unserialize(data, parent=parent)
+
+    @staticmethod
+    def required(*params):
+        def decor(f):
+            def wrapper(cls, data, **kw):
+                if any(not x in data for x in params):
+                    raise DirectoryDistributive(
+                        _("Failed to unserialize %s") % cls.__class__.__name__)
+                return f(cls,data, **kw)
+
+            return wrapper
+
+        return decor
+
+    def restore(self):
+        raise DistributiveError(_("Recovery is not implemented"))
+
+    def is_invalid(self):
+        try:
+            return not "cl_make_profile" in self.getInfo()
+        except DistributiveError:
+            return True
+
 class DirectoryDistributive(Distributive):
+    """
+    Дистрибутив в директории
+    """
+    data = [{'name': 'proc',
+             'type': 'proc',
+             'target': 'proc',
+             'source': 'none'},
+            {'name': 'sys',
+             'type': 'sysfs',
+             'target': 'sys',
+             'source': 'none'},
+            {'name': 'dev',
+             'type': 'bind',
+             'target': 'dev',
+             'source': '/dev'},
+            {'name': 'devpts',
+             'type': 'bind',
+             'target': 'dev/pts',
+             'source': '/dev/pts'},
+            {'name': 'remote',
+             'type': 'bind',
+             'target': 'var/calculate/remote',
+             'source': '/var/calculate/remote'},
+            ]
+
     def __init__(self,directory,parent=None,mdirectory=None):
         Distributive.__init__(self,parent=parent)
         self.directory = directory
         self.mdirectory = mdirectory
+        self.system_mounted = False
         if not parent:
             self._makeDirectory(self.directory)
 
+    def mountSystemDirectories(self, skip=("remote",)):
+        """
+        Подключить к дистрибутиву системые ресурсы (/proc, /sys)
+        :return:
+        """
+        for obj in filter(lambda x: x['name'] not in skip, self.data):
+            target_path = path.join(self.directory, obj['target'])
+            if obj['type'] == 'bind':
+                self._mountToBind(obj['source'], target_path)
+            else:
+                self._mountToDirectory(obj['source'],target_path,
+                                       "-t %s"%obj['type'])
+        self.system_mounted = True
+
+    def umountSystemDirectories(self):
+        for obj in reversed(self.data):
+            target_path = path.join(self.directory, obj['target'])
+            if isMount(target_path):
+                self._umountDirectory(target_path)
+        self.system_mounted = False
+
     def getType(self):
         return _("directory '%s'")%self.directory
 
@@ -502,6 +615,11 @@ class DirectoryDistributive(Distributive):
             self._removeDirectory(child.directory)
             child.directory = None
 
+    def close(self):
+        if self.system_mounted:
+            self.umountSystemDirectories()
+        Distributive.close(self)
+
     def convertToDirectory(self):
         if self.mdirectory:
             return self.bindDirectory(self.mdirectory)
@@ -513,11 +631,11 @@ class DirectoryDistributive(Distributive):
         execStr = '/bin/rm -rf --one-file-system %s'%self.directory
         res,errmes = self.runOsCommand(execStr)
         if res == 0:
+            self._makeDirectory(self.directory)
             return True
         else:
-            raise DistributiveError(_("Failed to format the partition") +
-                                    " %s:\n%s"%(dev,errmes))
-        self._makeDirectory(self.directory)
+            raise DistributiveError(_("Failed to clean directory") +
+                                    " %s:\n%s" % (self.directory, errmes))
 
     def installFrom(self, source, **kwargs):
         """Install distributive to directory from source distributive"""
@@ -529,6 +647,27 @@ class DirectoryDistributive(Distributive):
             # copy distributive from source to this
             self.rsync(dFrom.directory,self.directory,**kwargs)
 
+    def serialize(self):
+        d = {'type': Distributive.Type.Directory,
+             'directory': self.directory,
+             'system_mounted':self.system_mounted,
+             'childs': [x.serialize() for x in self.childs]}
+        if self.mdirectory:
+            d['mdirectory'] = self.mdirectory
+        return d
+
+    @classmethod
+    @Distributive.required("directory", "system_mounted")
+    def unserialize(self, data, parent=None):
+        ld = DirectoryDistributive(data['directory'],parent=parent)
+        ld.system_mounted = data['system_mounted']
+        if "mdirectory" in data:
+            ld.mdirectory = data['mdirectory']
+        ld.childs = [Distributive.unserialize(x, parent=ld) for x in
+                     data.get('childs', [])]
+        return ld
+
+
 class DataPartition:
     """Data partition"""
     dev = None
@@ -600,19 +739,19 @@ class MultiPartitions:
 
 
 class PartitionDistributive(Distributive):
-    formatUtilities = { 'ext2':'/sbin/mkfs.ext2 %s %s',
-                        'ext3':'/sbin/mkfs.ext3 %s %s',
-                        'ext4':'/sbin/mkfs.ext4 %s %s',
-                        'jfs':'/sbin/mkfs.jfs %s -f %s',
-                        'reiserfs':'/sbin/mkfs.reiserfs %s -f %s',
-                        'btrfs':'/sbin/mkfs.btrfs -f %s %s',
-                        'nilfs2':'/sbin/mkfs.nilfs2 %s %s',
-                        'xfs':'/sbin/mkfs.xfs %s -f %s',
-                        'vfat':'/usr/sbin/mkfs.vfat %s -F 32 %s',
-                        'ntfs-3g':'/usr/sbin/mkfs.ntfs %s -FQ %s',
-                        'ntfs':'/usr/sbin/mkfs.ntfs %s -FQ %s',
-                        'uefi':'/usr/sbin/mkfs.vfat %s %s',
-                        'swap':'/sbin/mkswap %s'
+    formatUtilities = {'ext2':'/sbin/mkfs.ext2 %s %s',
+                       'ext3':'/sbin/mkfs.ext3 %s %s',
+                       'ext4':'/sbin/mkfs.ext4 %s %s',
+                       'jfs':'/sbin/mkfs.jfs %s -f %s',
+                       'reiserfs':'/sbin/mkfs.reiserfs %s -f %s',
+                       'btrfs':'/sbin/mkfs.btrfs -f %s %s',
+                       'nilfs2':'/sbin/mkfs.nilfs2 %s %s',
+                       'xfs':'/sbin/mkfs.xfs %s -f %s',
+                       'vfat':'/usr/sbin/mkfs.vfat %s -F 32 %s',
+                       'ntfs-3g':'/usr/sbin/mkfs.ntfs %s -FQ %s',
+                       'ntfs':'/usr/sbin/mkfs.ntfs %s -FQ %s',
+                       'uefi':'/usr/sbin/mkfs.vfat %s %s',
+                       'swap':'/sbin/mkswap %s'
     }
     labelForUtilities = { 'ext2':'-L %s',
                         'ext3':'-L %s',
@@ -922,6 +1061,25 @@ class PartitionDistributive(Distributive):
         # install into directroy distributive from source
         distrTo.installFrom(source,**kwargs)
 
+    def serialize(self):
+        d = {'type': Distributive.Type.Partition,
+             'partition': self.partition,
+             'mdirectory': self.mdirectory,
+             'flag_remove_directory': self.flagRemoveDir,
+             'childs': [x.serialize() for x in self.childs]}
+        return d
+
+    @classmethod
+    @Distributive.required("partition", "mdirectory", "flag_remove_directory")
+    def unserialize(self, data, parent=None):
+        ld = PartitionDistributive(data['directory'], parent=parent)
+        ld.mdirectory = data['mdirectory']
+        ld.flagRemoveDir = data['flag_remove_directory']
+        ld.childs = [Distributive.unserialize(x, parent=ld) for x in
+                     data.get('childs', [])]
+        return ld
+
+
 class ArchiveDistributive(Distributive):
     def __init__(self,file,parent=None,mdirectory="/var/calculate/tmp/stage"):
         Distributive.__init__(self,parent=parent)
@@ -1016,6 +1174,22 @@ class ArchiveDistributive(Distributive):
         # install into directroy distributive from source
         self.packToArchive(dFrom.directory, self.file)
 
+    def serialize(self):
+        d = {'type': Distributive.Type.Archive,
+             'file': self.file,
+             'mdirectory':self.mdirectory,
+             'childs': [x.serialize() for x in self.childs]}
+        return d
+
+    @classmethod
+    @Distributive.required("mdirectory","file")
+    def unserialize(self, data, parent=None):
+        ld = ArchiveDistributive(data['file'],
+                                mdirectory=data['mdirectory'], parent=parent)
+        ld.childs = [Distributive.unserialize(x, parent=ld)
+                     for x in data.get('childs',[])]
+        return ld
+
 class SquashDistributive(Distributive):
     def __init__(self,file,parent=None,mdirectory="/mnt/livecd",exclude=None,
                  compress=""):
@@ -1078,6 +1252,23 @@ class SquashDistributive(Distributive):
         # install into directroy distributive from source
         self.packToSquash(dFrom.directory, self.file)
 
+    def serialize(self):
+        d = {'type': Distributive.Type.SquashFS,
+             'file': self.file,
+             'mdirectory':self.mdirectory,
+             'childs': [x.serialize() for x in self.childs]}
+        return d
+
+    @classmethod
+    @Distributive.required("mdirectory","file")
+    def unserialize(self, data, parent=None):
+        ld = SquashDistributive(data['file'],
+                             mdirectory=data['mdirectory'], parent=parent)
+        ld.childs = [Distributive.unserialize(x, parent=ld)
+                     for x in data.get('childs',[])]
+        return ld
+
+
 class IsoDistributive(Distributive):
     def __init__(self,file,parent=None,mdirectory="/mnt/cdrom",
             bdirectory="/var/calculate/tmp/iso",exclude=None,
@@ -1237,6 +1428,25 @@ class IsoDistributive(Distributive):
             if path.lexists(self.bdirectory) and self.file != self.bdirectory:
                 self._removeDirectory(self.bdirectory)
 
+    def serialize(self):
+        d = {'type': Distributive.Type.Iso,
+             'file': self.file,
+             'mdirectory':self.mdirectory,
+             'bdirectory':self.bdirectory,
+             'childs': [x.serialize() for x in self.childs]}
+        return d
+
+    @classmethod
+    @Distributive.required("mdirectory","bdirectory","file")
+    def unserialize(self, data, parent=None):
+        ld = IsoDistributive(data['file'],
+                             mdirectory=data['mdirectory'],
+                             bdirectory=data['bdirectory'],
+                             parent=parent)
+        ld.childs = [Distributive.unserialize(x, parent=ld)
+                     for x in data.get('childs',[])]
+        return ld
+
 class FlashDistributive(PartitionDistributive):
     def _checkMount(self,dev):
         """Checking mount point"""
@@ -1450,3 +1660,105 @@ class PxeDistributive(Distributive):
             raise DistributiveError(
                 _("PXE install does not support %s"%
                     source.__class__.__name__))
+
+
+class LayeredDistributive(Distributive):
+    """
+    Каталог дистрибутива для сборки
+    """
+    needFormat = False
+
+    def __init__(self, mdirectory, diff_directory, image_file=None,
+                 parent=None):
+        """
+        :param mdirectory: результирующий каталог
+        :param diff_directory: каталог содержит изменения от оригинала
+        :param image_file: образ оригинала
+        :param parent: родительский дистрибутив
+        """
+        super(LayeredDistributive, self).__init__(parent=parent)
+        self.mdirectory = mdirectory
+        self.diff_directory = diff_directory
+        self.image_mount_dir = None
+        self.image_distro = None
+        self.image_file = image_file
+
+    def getType(self):
+        return _("layered '%s %s'")%(self.image_file, self.diff_directory)
+
+    def clearDiff(self):
+        if path.exists(self.diff_directory):
+            self._removeDirectory(self.diff_directory)
+
+    def _mountLayers(self, target):
+        """Mount squashfs to directory"""
+        self._makeDirectory(target)
+        if not path.exists(self.diff_directory):
+            self._makeDirectory(self.diff_directory)
+        self._mountToDirectory("none", target, mountopts=(
+            "-t aufs -o udba=reval,br:%(work)s=rw:%(static)s=ro" %
+            {"work": self.diff_directory,
+             "static": self.image_mount_dir}))
+
+    def _umountLayers(self, directory):
+        self._umountDirectory(directory)
+        self._removeDirectory(directory)
+
+    def _mountLiveImage(self):
+        """Mount squashfs to directory"""
+        self.image_distro = IsoDistributive(file=self.image_file, parent=self)
+        self.image_mount_dir = (
+            self.image_distro.convertToDirectory().getDirectory())
+
+    def _umountLiveImage(self):
+        if self.image_distro:
+            self.image_distro.close()
+            self.image_distro = None
+
+    def convertToDirectory(self):
+        """Convert scrach directories to one directory"""
+        mdirectory = self.mdirectory
+        for child in self.childs:
+            if isinstance(child,DirectoryDistributive) and \
+                            mdirectory in child.directory:
+                return child
+        mdirectory = self._getMntDirectory(mdirectory)
+        self._mountLiveImage()
+        self._mountLayers(mdirectory)
+        return DirectoryDistributive(mdirectory, parent=self)
+
+    def releaseChild(self, child):
+        """Umount child Directory distributive"""
+        if isinstance(child,DirectoryDistributive):
+            self._umountLayers(child.directory)
+            self._umountLiveImage()
+            child.directory = None
+
+    def installFrom(self, source, **kwargs):
+        """Install distributive to partition from source distributive"""
+        # make temporary directory for creating iso image
+        if isinstance(source, (IsoDistributive, SquashDistributive)):
+            self.image_file = source.file
+            return True
+        raise Distributive("Not implementation")
+
+    def serialize(self):
+        d = {'type': Distributive.Type.Layered,
+                'mdirectory': self.mdirectory,
+                'diff_directory': self.diff_directory,
+                'image_file': self.image_file,
+                'childs': [x.serialize() for x in self.childs]}
+        if self.image_mount_dir:
+            d['image_mount_dir'] = self.image_mount_dir
+        return d
+
+    @classmethod
+    @Distributive.required("mdirectory", "diff_directory", "image_file")
+    def unserialize(self, data, parent=None):
+        ld = LayeredDistributive(data['mdirectory'],
+                                 data['diff_directory'],
+                                 data['image_file'],
+                                 parent=parent)
+        ld.childs = [Distributive.unserialize(x, parent=ld)
+                     for x in data.get('childs',[])]
+        return ld

install/variables/distr.py

@@ -249,7 +249,7 @@ class VariableClImage(ReadonlyVariable):
     def get(self):
         """Get image file from distributive repository"""
         try:
-            if self.Get('cl_action') != 'system':
+            if not self.Get('cl_action') in ('system','prepare'):
                 return Distributive.fromFile('/')
             filename = self.Get('cl_image_filename')
             if filename: