# -*- coding: utf-8 -*- # Copyright 2010-2016 Mir Calculate. http://www.calculate-linux.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from os import path from random import choice import string import os from time import sleep import datetime import re import sys import operator import json from shutil import copyfile, copytree from subprocess import Popen, PIPE from itertools import * from functools import partial from calculate.lib.datavars import VariableError from calculate.lib.utils.mount import isMount, Btrfs, BtrfsError from calculate.lib.utils.files import (removeDir, processProgress, STDOUT, typeFile, pathJoin, process, listDirectory, checkUtils, MAGIC_COMPRESS, MAGIC_SYMLINK, MAGIC_CONTINUE, makeDirectory, isEmpty, check_rw, calculate_exclude, PercentProgress, getProgPath, xztaropen) import calculate.lib.utils.device as device from calculate.lib.utils.device import (detectDeviceForPartition, countPartitions) from calculate.lib.utils.tools import classproperty, Signal, traverse from calculate.lib.utils.text import _u8 from calculate.lib.variables.linux import LinuxDataVars, Linux from calculate.lib.cl_lang import setLocalTranslate, _ from functools import reduce setLocalTranslate('cl_install3', sys.modules[__name__]) class DefaultMountPath(object): """ Пути по умолчанию для монтирования образов """ BaseMountPath = '/run/calculate/mount' BuildDirectory = '/var/calculate/tmp/iso' @classproperty def IsoImage(cls): return path.join(cls.BaseMountPath, "iso") @classproperty def SquashImage(cls): return path.join(cls.BaseMountPath, "squash") @classproperty def ArchiveImage(cls): return path.join(cls.BaseMountPath, "tarball") @classproperty def DefaultMount(cls): return path.join(cls.BaseMountPath, "distro") @classproperty def InstallMount(cls): return path.join(cls.BaseMountPath, "install") class cpProcessProgress(processProgress): def init(self, *args, **kwargs): self.maxfiles = kwargs.get('maxfiles', 1) self.stderr = STDOUT self.command.append("-dRv") def processInit(self): self.value = 0 self.percent = 0 return 0 def processEnd(self): self.value = self.maxfiles self.percent = 100 return 100 def processString(self, strdata): if strdata.startswith("cp:") or strdata.startswith("/bin/cp:"): self._cachedata.append(strdata.partition(": ")[2]) if "->" in strdata: self.value += 1 if self.maxfiles: percent = 100 * self.value / self.maxfiles percent = min(percent, 99) if percent > self.percent: self.percent = percent return percent else: return None def progressCopyFile(source, dest): """ Copy file with progress """ size = int(os.lstat(source).st_size) bufsize = (100 - (size % 100) + size) / 100 with open(source, 'rb') as infile: with open(dest, 'w') as outfile: for i in xrange(1, 101): outfile.write(infile.read(bufsize)) yield i class DistributiveError(Exception): """Error for distributive operations""" pass class Distributive(object): """ Distributive object. Parent object for all distributive. """ class Type(object): Directory = "dir" Partition = "part" SquashFS = "squash" Archive = "arch" Iso = "iso" Layered = "layered" mountError = _("Failed to mount") + " %s:\n%s" reLive = re.compile(r"^live[^.]*\.squashfs(\.(\d+))?$", re.S) contentCache = {} needFormat = True def __init__(self, parent=None): self.childs = [] self.ref = 0 self.locked = False # if specified parent if isinstance(parent, Distributive): # save parent type for correct resource release self.parent = parent # append this object as child to specified parent parent.childs.append(self) else: self.parent = None @classmethod def fromFile(cls, filename): """Get Distributive object by filename""" # MAGIC_COMPRESS 0x000004 Check inside compressed files tf = typeFile(magic=MAGIC_COMPRESS | MAGIC_SYMLINK | MAGIC_CONTINUE) ftype = tf.getMType(filename) if ftype: if "block special" in ftype: for distrType in (IsoDistributive, FlashDistributive, PartitionDistributive): distr = distrType(filename) res = distr.probe() distr.close() if res: return distr if "ISO 9660 CD-ROM" in ftype: return IsoDistributive(filename) elif "7-zip" in ftype or \ "POSIX tar archive" in ftype: if "rootfs.tar" in filename: return ContainerDistributive(path.dirname(filename)) else: return ArchiveDistributive(filename) elif "Squashfs filesystem" in ftype: return SquashDistributive(filename) elif path.isdir(filename): if path.isfile(path.join(filename, "rootfs.tar.xz")): return ContainerDistributive(filename) elif path.isfile(path.join(filename, "livecd")): return IsoDistributive(filename) else: important_dirs = ("etc", "lib", "bin", "sbin", "var") if all(path.exists(path.join(filename, x)) for x in important_dirs): return DirectoryDistributive(filename) raise DistributiveError(_("Wrong distribution") + " '%s':\n%s" \ % (filename, ftype)) def getType(self): return _("empty") def detach(self): """Detach child distributive from parent. At example: ArchiveDistributive create child DirectoryDistributive by unpacking to temporary directory and at close method remove it. If the removing directroy do not need, then need call detach in DirectoryDistributive object dist1 = ArchiveDistributive(file="/os.tar.bz2",mdirectory="/tmp/test") dist2 = dist1.convertToDirectory() dist2.detach() dist1.close() ... """ self.parent = None def reserve(self): self.locked = True def release(self): self.locked = False def __enter__(self): self.ref += 1 return self def __exit__(self, type, value, traceback): self.ref -= 1 if not self.ref: self.close() def close(self): """Release all child distributive and release himself. Need call this method at end work with object for right releasing resources. Example: dist1 = PartitionDistributive(partition="/dev/sda2") dist2 = dist1.convertToDirectory() dist1.close() """ # print "Close", self, self.locked if self.locked: return False if self.ref: self.ref -= 1 return False else: try: # close all child if self.childs: for child in reversed(self.childs): # check detach if child.parent: child.close() self.childs = [] # if has parent if self.parent: self.parent.releaseChild(self) finally: self.parent = None self.childs = [] return True def releaseChild(self, child): """Method of release child state of distributive At example: PartitionDistributive may be DirectoryDistributive by mounting it to directory. But at end this directory must be unmounted.""" pass def convertToDirectory(self): """Default c raise error about impossible convert object""" raise DistributiveError(_("Failed to convert") + " '%s' " \ % self.__class__.__name__ + _("to") + \ " '%s'" % "DirectoryDistributive") # def __del__(self): # """Uncomment this method for automaticaly release all distributive # instance""" # self.close() @staticmethod def clear_empty_directories(dn): dn = path.dirname(dn) while dn and path.exists(dn) and isEmpty(dn) and not isMount(dn): os.rmdir(dn) dn = path.dirname(dn) def getDirectory(self): """Get directory which contains distro""" return self.convertToDirectory().directory def getBootDirectory(self): """Get directory which contains boot""" return path.join(self.getDirectory(), "boot") def getEfiDirectory(self): """Get directory which contains boot/efi""" return self.getEfiDirectories()[0] def getEfiDirectories(self): return [path.join(self.getDirectory(), "boot/efi")] def _makeDirectory(self, pathname): """Make directory and parent. If directory exists then return False else True""" try: parent = path.split(path.normpath(pathname))[0] if not path.exists(parent): self._makeDirectory(parent) else: if path.exists(pathname): return False os.mkdir(pathname) return True except Exception as e: raise DistributiveError(_("Failed to create the directory") + " '%s':\n%s" % (pathname, str(e))) except KeyboardInterrupt as e: raise DistributiveError(_("Failed to create the directory") + " '%s':\n%s" % (pathname, str(e))) def get_squash_size(self): """ Получить размер squash образа :return: """ raise DistributiveError(_("Squash size unsupported for %s") % str(self.__class__.__name__)) def _removeDirectory(self, directory): """Remove directory and files contained in it""" try: if path.exists(directory): removeDir(directory) except Exception as e: raise DistributiveError(_("Failed to remove the directory from") + \ " '%s':\n%s" % (directory, str(e))) except KeyboardInterrupt as e: raise DistributiveError(_("Failed to remove the directory from") + \ " '%s':\n%s" % (directory, str(e))) def _copyfile(self, infile, outfile): try: copyfile(infile, outfile) except Exception as e: raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s" \ % (infile, outfile, str(e))) def _copytree(self, indir, outdir): try: copytree(indir, outdir, symlinks=True) except Exception as e: raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s" \ % (indir, outdir, str(e))) except KeyboardInterrupt as e: raise DistributiveError(_("Failed to copy") + " '%s' to '%s':\n%s" \ % (indir, outdir, str(e))) def rsync(self, fromdir, todir, callbackProgress=None, byfile=None, filesnum=0, noperm=False, **kwargs): """Copy files from 'fromdir' directory to 'todir' directory""" cpCmd = getProgPath('/bin/cp') if not cpCmd: raise DistributiveError(_("'%s' not found") % "cp") try: joinFrom = partial(path.join, fromdir) params = [cpCmd, "-x"] + \ (["--no-preserve=mode,ownership"] if noperm else ["-a"]) + \ list(map(joinFrom, filterfalse(byfile or operator.not_, listDirectory(fromdir)))) + \ [todir] cpProcess = cpProcessProgress(*params, maxfiles=filesnum, stderr=STDOUT) for perc in cpProcess.progress(): if callbackProgress and not byfile: callbackProgress(perc) res = cpProcess.success() errmes = cpProcess.read() # copy by one file if byfile: percFiles = list(filter(byfile, listDirectory(fromdir))) if len(percFiles) > 1: maxPercOnFile = 100 / len(percFiles) recountPerc = \ lambda perc, num: perc / len( percFiles) + maxPercOnFile * num else: recountPerc = lambda perc, num: perc for i, copyfile in enumerate(percFiles): for perc in progressCopyFile(joinFrom(copyfile), path.join(todir, copyfile)): if callbackProgress: callbackProgress(recountPerc(perc, i)) except Exception as e: res = False errmes = str(e) if not res: raise DistributiveError(_("Failed to copy files from") + \ " '%s' " % fromdir + _("to") + \ " '%s':\n%s" % (todir, errmes)) def _mountToBind(self, srcDirectory, destDirectory): """Mount srcDirectory to destDirectory""" mount = process('/bin/mount', "-o", "bind", srcDirectory, destDirectory, stderr=STDOUT) if mount.success(): return True else: try: os.rmdir(destDirectory) except OSError: pass raise DistributiveError( self.mountError % (srcDirectory, mount.read())) def performFormat(self): pass def formatPartition(self, dev, format="ext4", label="", purpose=None): pass def rndString(self): """Get random string with len 8 char""" return "".join([choice(string.ascii_letters + string.digits) for i in xrange(0, 8)]) def _getSquashNum(self, reMatch): if reMatch.groups()[1] and reMatch.groups()[1].isdigit(): return int(reMatch.groups()[1]) else: return 0 def _getLastLive(self, directory): """Get last live squashfs image from directory""" squashfiles = list(filter(lambda x: x, map(self.reLive.search, listDirectory(directory)))) if squashfiles: return max(squashfiles, key=self._getSquashNum).group() return None def _mountToDirectory(self, file, directory, mountopts="", count=2): """Mount squashfs to directory""" # 2816 code return by mount if device is absent (update /dev by udev) NO_SUCH_DEVICE = 2816 if isMount(directory): raise DistributiveError( _("Failed to mount to the directory: %s\n") % directory + _("Directory already mounted")) mountopts_list = filter(lambda x: x, mountopts.split(" ")) mountProcess = process('/bin/mount', file, directory, *mountopts_list, stderr=STDOUT) if mountProcess.success(): return True else: # try mount 3 times with interval 0.5 second if mountProcess.returncode() == NO_SUCH_DEVICE and count: sleep(0.5) mountProcess.close() return self._mountToDirectory(file, directory, mountopts, count - 1) try: self._removeDirectory(directory) except Exception: pass raise DistributiveError( self.mountError % (file, mountProcess.read())) def _umountDirectory(self, directory): """Umount directory""" if isMount(directory): processUmount = None for wait in [0, 0.5, 2, 5]: sleep(wait) processUmount = process('/bin/umount', directory, stderr=STDOUT) if processUmount.success(): return True else: raise DistributiveError(_("Failed to umount") + " %s:\n%s" % (directory, processUmount.read())) else: return True def _getMntDirectory(self, directory): """Get directory name, which will use for mounting or unpacking If queried name is not free then to name append random string """ newDirectoryName = directory for i in range(2, 9999): if not path.exists(newDirectoryName): return newDirectoryName else: newDirectoryName = "%s.%02d" % (directory, i) return newDirectoryName @staticmethod def getInfoFromDirectory(directory): d = {} try: if path.lexists(path.join(directory, 'lib64')): d['os_arch_machine'] = 'x86_64' elif path.lexists(path.join(directory, 'lib')): d['os_arch_machine'] = 'i686' else: d['os_arch_machine'] = '' dv = LinuxDataVars(systemRoot=directory) d["os_linux_shortname"] = dv.Get('os_linux_shortname') d['os_linux_ver'] = dv.Get('os_linux_ver') d['os_linux_build'] = dv.Get('os_linux_build') d['os_linux_name'] = dv.Get('os_linux_name') d['os_linux_subname'] = dv.Get('os_linux_subname') d['os_linux_system'] = dv.Get('os_linux_system') d['cl_make_profile'] = dv.Get('cl_make_profile') d['cl_profile_name'] = dv.Get('cl_profile_name') # make lazy call d['os_linux_files'] = partial(dv.Get, 'os_linux_files') d['os_chrootable_set'] = "off" try: if process("/usr/bin/chroot", directory, "/bin/true").success(): d['os_chrootable_set'] = "on" except Exception: pass except VariableError: pass return d.copy() def getInfo(self, filename=None): """Get info from content""" image = None try: # get from cache keyCache = None if not filename: for keyname in ("file", "partition", "directory"): if hasattr(self, keyname): keyCache = getattr(self, keyname) elif filename in self.contentCache: keyCache = filename if keyCache and keyCache in self.contentCache: return Distributive.contentCache[keyCache].copy() distr = None # may be directory is isodir (directory which contains iso image) extname = "isodir" try: distr = self.fromFile(filename) if filename else self mapExtName = {DirectoryDistributive: "dir", IsoDistributive: "isodir", FlashDistributive: "flash", ContainerDistributive: "lxc", PartitionDistributive: "partdir"} extname = mapExtName.get(distr.__class__, "") if isinstance(distr, ContainerDistributive): d = distr.get_information() d['ext'] = extname return d.copy() if isinstance(distr, ArchiveDistributive): raise DistributiveError("Not for archive distributive") image = distr.convertToDirectory() except Exception as e: # TODO: отладка почему образ не подходит #print str(e) if distr: distr.close() return {}.copy() d = self.getInfoFromDirectory(image.directory) d['ext'] = extname if distr: distr.close() if keyCache and not path.isdir(keyCache): Distributive.contentCache[keyCache] = d return d.copy() finally: if image: image.close() @classmethod def unserialize(cls, data, parent=None): class_mapper = {cls.Type.Directory: DirectoryDistributive, cls.Type.Iso: IsoDistributive, cls.Type.Partition: PartitionDistributive, cls.Type.SquashFS: SquashDistributive, cls.Type.Layered: LayeredDistributive} if not data.get('type', '') in class_mapper: raise DistributiveError(_("Failed to unserialize type %s") % data.get('type', '')) return class_mapper[data['type']].unserialize(data, parent=parent) @staticmethod def required(*params): def decor(f): def wrapper(cls, data, **kw): if any(not x in data for x in params): raise DirectoryDistributive( _("Failed to unserialize %s") % cls.__class__.__name__) return f(cls, data, **kw) return wrapper return decor def restore(self): raise DistributiveError(_("Recovery is not implemented")) def is_invalid(self): try: return not "cl_make_profile" in self.getInfo() except DistributiveError: return True def post_clear(self): return True class DirectoryDistributive(Distributive): """ Дистрибутив в директории """ data = [{'name': 'proc', 'type': 'proc', 'target': 'proc', 'source': 'none'}, {'name': 'sys', 'type': 'sysfs', 'target': 'sys', 'source': 'none'}, {'name': 'dev', 'type': 'bind', 'target': 'dev', 'source': '/dev'}, {'name': 'devpts', 'type': 'bind', 'target': 'dev/pts', 'source': '/dev/pts'}, {'name': 'remote', 'type': 'bind', 'target': 'var/calculate/remote', 'source': '/var/calculate/remote'}, ] def __init__(self, directory, parent=None, mdirectory=None): Distributive.__init__(self, parent=parent) self.no_unmount = False self.directory = directory self.mdirectory = mdirectory self.system_mounted = False if not parent: self._makeDirectory(self.directory) def hasSystemDirectories(self): return self.system_mounted or self.directory == '/' def mountSystemDirectories(self, skip=("remote",)): """ Подключить к дистрибутиву системые ресурсы (/proc, /sys) :return: """ if not self.system_mounted: for obj in filter(lambda x: x['name'] not in skip, self.data): target_path = path.join(self.directory, obj['target']) if obj['type'] == 'bind': if not path.exists(target_path): self._makeDirectory(target_path) if not path.exists(obj['source']): self._makeDirectory(obj['source']) self._mountToBind(obj['source'], target_path) else: self._mountToDirectory(obj['source'], target_path, "-t %s" % obj['type']) self.system_mounted = True def umountSystemDirectories(self): for obj in reversed(self.data): target_path = path.join(self.directory, obj['target']) if isMount(target_path): self._umountDirectory(target_path) self.system_mounted = False def getType(self): return _("directory '%s'") % _u8(self.directory) def bindDirectory(self, mdirectory): for child in self.childs: if isinstance(child, DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._makeDirectory(mdirectory) self._mountToBind(self.directory, mdirectory) return DirectoryDistributive(mdirectory, parent=self) def releaseChild(self, child): """Remove child Directory distributive""" if isinstance(child, DirectoryDistributive): self._umountDirectory(child.directory) self._removeDirectory(child.directory) child.directory = None def close(self): if not self.locked and self.system_mounted: self.umountSystemDirectories() Distributive.close(self) def convertToDirectory(self): if self.mdirectory: return self.bindDirectory(self.mdirectory) else: return self def performFormat(self): """Format for directory - removing all files""" rm_command = getProgPath("/bin/rm") p = process(rm_command, "-rf", "--one-file-system", self.directory, stderr=process.STDOUT) if p.success(): self._makeDirectory(self.directory) return True else: raise DistributiveError(_("Failed to clean directory") + " %s:\n%s" % (self.directory, p.read())) def post_clear(self): if path.exists(self.directory): rm_command = getProgPath("/bin/rm") p = process(rm_command, "-rf", "--one-file-system", self.directory) p.success() self.clear_empty_directories(self.directory) return True def installFrom(self, source, **kwargs): """Install distributive to directory from source distributive""" if isinstance(source, ArchiveDistributive): source.unpackTo(self.directory) else: # get source distributive as directory distributive dFrom = source.convertToDirectory() # copy distributive from source to this self.rsync(dFrom.directory, self.directory, **kwargs) def serialize(self): d = {'type': Distributive.Type.Directory, 'directory': self.directory, 'system_mounted': self.system_mounted, 'childs': [x.serialize() for x in self.childs]} if self.mdirectory: d['mdirectory'] = self.mdirectory return d @classmethod @Distributive.required("directory", "system_mounted") def unserialize(cls, data, parent=None): ld = DirectoryDistributive(_u8(data['directory']), parent=parent) ld.system_mounted = _u8(data['system_mounted']) if "mdirectory" in data: ld.mdirectory = _u8(data['mdirectory']) ld.childs = [Distributive.unserialize(x, parent=ld) for x in data.get('childs', [])] return ld class DataPartition(object): """Data partition""" dev = None mountPoint = None fileSystem = "ext4" isFormat = False systemId = None partitionTable = None class MultiPartitions: """Data partition list""" def __init__(self): self.listPartitions = [] def addPartition(self, **argv): """Add partition in data partition list""" dictDataPart = reduce(lambda x, y: \ x.update({y: getattr(DataPartition, y)}) or x, filter(lambda x: not x.startswith('_'), DataPartition.__dict__), {}) updateAttrData = filter(lambda x: x[1] is not None, dictDataPart.items()) defaultAttr = [] for attrName, attrValue in updateAttrData: if not attrName in argv.keys(): defaultAttr.append(attrName) argv[attrName] = attrValue if set(argv.keys()) != set(dictDataPart.keys()): notFoundAttr = set(dictDataPart.keys()) - set(argv.keys()) if notFoundAttr: raise DistributiveError(_("The following attributes " "are not specified: (%s)") \ % ", ".join( map(lambda x: "DataPartition.%s" % x, notFoundAttr))) unnecessaryAttr = (set(dictDataPart.keys()) ^ set(argv.keys())) - \ set(dictDataPart.keys()) if unnecessaryAttr: raise DistributiveError(_("Failed to use attributes (%s) ") \ % ", ".join( map(lambda x: "DataPartition.%s" % x, unnecessaryAttr))) else: partObj = DataPartition() for attr, value in argv.items(): if attr in defaultAttr: continue setattr(partObj, attr, value) self.listPartitions.append(partObj) def getSystemId(self): """Get systemID for change [None,82,...]""" return list(map(lambda x: x.systemId, self.listPartitions)) def getPartitionTable(self): """Get systemID for change [dos,gpt,...]""" return list(map(lambda x: x.partitionTable, self.listPartitions)) def getIsFormat(self): """Get list is format [True,...]""" return list(map(lambda x: x.isFormat, self.listPartitions)) def getFileSystems(self): """Get list filesystems ["reiserfs",...]""" return list(map(lambda x: x.fileSystem, self.listPartitions)) def getPartitions(self): """Get list partition ["/dev/sda",...]""" return list(map(lambda x: x.dev, self.listPartitions)) def getMountPoints(self): """Get list mount point ["/boot",...]""" return list(map(lambda x: x.mountPoint, self.listPartitions)) class FormatProcess(process): format_util = "" dos_id = "0" gpt_id = "0" def __init__(self, dev, label=None, purpose=None, compression=None): self.dev = dev self._label = label self.purpose = purpose self.compression = compression super(FormatProcess, self).__init__(*self.format_command()) def format_command(self): cmd = getProgPath(self.format_util) if not cmd: raise DistributiveError(_("command '%s' not found") % cmd) params = (self.bootparam() if self.purpose in ("root", "boot") else self.param()) return list(traverse( [cmd] + [x for x in params if x])) def postaction(self): return True def bootparam(self): return self.param() def param(self): return self.get_label(), self.dev def get_label(self): if not self._label: return () else: return self.label() def label(self): return () class FormatProcessGeneric(FormatProcess): dos_id = "83" gpt_id = "8300" def label(self): return "-L", self._label def param(self): return self.get_label(), self.dev class FormatExt2(FormatProcessGeneric): format_util = "/sbin/mkfs.ext2" class FormatExt3(FormatExt2): format_util = "/sbin/mkfs.ext3" class FormatExt4(FormatExt2): format_util = "/sbin/mkfs.ext4" class FormatJfs(FormatProcessGeneric): format_util = "/sbin/mkfs.jfs" def param(self): return self.get_label(), "-f", self.dev class FormatF2FS(FormatProcessGeneric): format_util = "/usr/sbin/mkfs.f2fs" def label(self): return "-l", self._label def param(self): return self.get_label(), "-f", self.dev class FormatReiserfs(FormatProcessGeneric): format_util = "/sbin/mkfs.reiserfs" def param(self): return self.get_label(), "-f", self.dev def label(self): return "-l", self._label class FormatNilfs2(FormatProcessGeneric): format_util = "/sbin/mkfs.nilfs2" class FormatXfs(FormatProcessGeneric): format_util = "/sbin/mkfs.xfs" def param(self): return self.get_label(), "-f", self.dev def nosparse(self): return "-i", "sparse=0" def bootparam(self): return self.get_label(), self.nosparse(), "-f", self.dev class FormatVfat(FormatProcess): dos_id = "0b" gpt_id = "0700" format_util = "/usr/sbin/mkfs.vfat" def param(self): return self.get_label(), "-F", "32", self.dev def label(self): return "-n", self._label class FormatNtfs(FormatProcessGeneric): dos_id = "07" gpt_id = "0700" format_util = "/usr/sbin/mkfs.ntfs" def param(self): return self.get_label(), "-FQ", self.dev class FormatUefi(FormatVfat): dos_id = "ef" gpt_id = "EF00" def label(self): return () class FormatSwap(FormatProcess): dos_id = "82" gpt_id = "8200" format_util = "/sbin/mkswap" def param(self): return self.dev, class FormatBtrfs(FormatProcessGeneric): format_util = "/sbin/mkfs.btrfs" def param(self): return "-f", self.get_label(), self.dev class PartitionDistributive(Distributive): format_map = { 'ext2': FormatExt2, 'ext3': FormatExt3, 'ext4': FormatExt4, 'jfs': FormatJfs, 'f2fs': FormatF2FS, 'reiserfs': FormatReiserfs, 'btrfs': FormatBtrfs, 'nilfs2': FormatNilfs2, 'xfs': FormatXfs, 'vfat': FormatVfat, 'ntfs-3g': FormatNtfs, 'ntfs': FormatNtfs, 'uefi': FormatUefi, 'btrfs': FormatBtrfs, 'btrfs-compress': FormatBtrfs, 'swap': FormatSwap } def __init__(self, partition, parent=None, mdirectory=None, check=False, multipartition=None, flagRemoveDir=True, fileSystem="ext4", isFormat=True, systemId=None, rootLabel="Calculate", partitionTable=None, compression=None): """Initialize partition distributive mdirectory - directory for mount check - check partition name and raise DistributiveError if partition has bad name """ Distributive.__init__(self, parent=parent) self.partition = partition self.fileSystem = fileSystem self.mdirectory = mdirectory or DefaultMountPath.DefaultMount self.multipartition = multipartition self.flagRemoveDir = flagRemoveDir self.isFormat = isFormat self.DirectoryObject = None self.systemId = systemId self.rootLabel = rootLabel self.compression = compression self.partitionTable = partitionTable def getType(self): return _("partition '%s'") % self.partition def probe(self): """Check directory for flash content""" try: pathname = self.getDirectory() except Exception: return False return Linux().detectOtherShortname(pathname) def _mountPartition(self, partition, directory, opts=""): """Mount partition to directory""" self._makeDirectory(directory) if "ntfs" in opts: source_dir = isMount(partition) if source_dir: self._mountToBind(source_dir, directory) return self._mountToDirectory(partition, directory, opts) def _umountPartition(self, directory): """Umount partition and remove directory""" self._umountDirectory(directory) if self.flagRemoveDir: self._removeDirectory(directory) def releaseChild(self, child): """Umount child Directory distributive""" if isinstance(child, DirectoryDistributive): self._umountPartition(child.directory) child.directory = None def _mountBind(self, srcDirectory, destDirectory): """Mount directory to directory""" self._makeDirectory(destDirectory) self._makeDirectory(srcDirectory) self._mountToBind(srcDirectory, destDirectory) def postinstallMountBind(self): """Mount bind mount point and create mount dirs""" if self.multipartition and self.DirectoryObject: mulipartDataBind = list(filter(lambda x: x[2] == "bind", self.getMultipartData())) dirObj = self.DirectoryObject mdirectory = dirObj.directory for srcDir, destDir, fileSystem, isFormat, partTable \ in mulipartDataBind: realDestDir = pathJoin(mdirectory, destDir) realSrcDir = pathJoin(mdirectory, srcDir) self._mountBind(realSrcDir, realDestDir) isFormat = False partObj = PartitionDistributive(realSrcDir, flagRemoveDir=False, fileSystem=fileSystem, isFormat=isFormat, parent=dirObj) DirectoryDistributive(realDestDir, parent=partObj) def getEfiDirectories(self): return [path.join(self.getDirectory(), x) for x in self.multipartition.getMountPoints() if x.startswith("/boot/efi") ] def getMultipartData(self): """Get multipartition data""" mulipartData = zip(self.multipartition.getPartitions(), self.multipartition.getMountPoints(), self.multipartition.getFileSystems(), self.multipartition.getIsFormat(), self.multipartition.getPartitionTable()) return mulipartData def convertToDirectory(self): """Convert partition to directory by mounting""" mapFS = {'btrfs-compress': 'btrfs'} mapOpts = {'btrfs-compress': ' -o compress=%s' % self.compression} mdirectory = self.mdirectory for child in self.childs: if isinstance(child, DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._mountPartition(self.partition, mdirectory, mapOpts.get(self.fileSystem,"")) dirObj = DirectoryDistributive(mdirectory, parent=self) if self.multipartition: mulipartDataNotBind = filter(lambda x: x[2] != "bind", self.getMultipartData()) for dev, mountPoint, fileSystem, isFormat, partTable \ in sorted(mulipartDataNotBind, key=lambda x: x[1]): realMountPoint = None if fileSystem != "swap": realMountPoint = pathJoin(mdirectory, mountPoint) self._mountPartition( dev, realMountPoint, "-t %s" % mapFS.get(fileSystem,fileSystem) + mapOpts.get(fileSystem,"")) partObj = PartitionDistributive(dev, flagRemoveDir=False, fileSystem=fileSystem, isFormat=isFormat, parent=dirObj) if realMountPoint is not None: DirectoryDistributive(realMountPoint, parent=partObj) self.DirectoryObject = dirObj return dirObj def formatAllPartitions(self): """Format all partitions""" FS, DEV, NEEDFORMAT, NEWID, PARTTABLE, MP = 0, 1, 2, 3, 4, 5 # get all information to matrix dataPartitions = zip(self.multipartition.getFileSystems() + \ [self.fileSystem], self.multipartition.getPartitions() + \ [self.partition], self.multipartition.getIsFormat() + \ [self.isFormat], self.multipartition.getSystemId() + \ [self.systemId], self.multipartition.getPartitionTable() + \ [self.partitionTable], self.multipartition.getMountPoints() + \ ["/"]) # get partition which need format formatPartitions = list(map(lambda x: (x[FS], x[DEV], x[NEWID], x[MP]), filter( lambda x: x[NEEDFORMAT] and x[FS] != "bind", dataPartitions))) # if has separate /boot partition bootmp = ["/boot", "/"] purpose_map = {"/boot": "boot", "/var/calculate": "calculate", "/": "root"} # format all get partition for fileSystem, dev, newID, mp in formatPartitions: if fileSystem == "swap": self.formatSwapPartition(dev) else: if newID == "EF00": fileSystem = "uefi" if dev == self.partition: self.formatPartition(dev, format=fileSystem, label=self.rootLabel, purpose=purpose_map.get(mp, None)) else: if mp == '/var/calculate': self.formatPartition(dev, format=fileSystem, label="Calculate") else: self.formatPartition(dev, format=fileSystem, purpose=purpose_map.get(mp, None)) # change system id for partitions changeidPartitions = list(map(lambda x: (x[NEWID], x[DEV], x[PARTTABLE]), filter(lambda x: x[NEWID], dataPartitions))) for systemid, dev, partTable in changeidPartitions: self.changeSystemID(dev, systemid, partTable) return True def _checkMount(self, dev): """Checking mount point""" if isMount(dev): raise DistributiveError( _("Failed to format %s: this partition is mounted") % dev) def formatPartition(self, dev, format="ext4", label="", purpose=None): """Format partition""" if not format in self.format_map: raise DistributiveError( _("The specified format of '%s' is not supported") % format) with open("/proc/swaps") as f: if dev in map(lambda y: y.split(" ")[0], filter(lambda x: x.startswith("/"),f)): raise DistributiveError( _("Failed to format %s: this partition is used as swap") % dev) self._checkMount(dev) if not os.access(dev, os.W_OK): raise DistributiveError(_("Failed to format the partition") + " %s:\n%s" % (dev, _("Permission denied"))) format_process = self.format_map[format](dev, label=label, purpose=purpose, compression=self.compression) if format_process.failed(): raise DistributiveError( _("Failed to format the partition") + " %s:\n%s" % (dev, format_process.readerr())) format_process.postaction() def performFormat(self): """Perform format for all partition of this distributive""" if self.multipartition: self.formatAllPartitions() elif self.isFormat: self.formatPartition(self.partition, format=self.fileSystem, label=self.rootLabel, purpose="root") if self.systemId: self.changeSystemID(self.partition, self.systemId, self.partitionTable) def changeSystemID(self, dev, systemid, parttable): """Change partition id, specified by systemid""" deviceName = detectDeviceForPartition(dev) if deviceName is None: raise DistributiveError( _("Failed to determine the parent device for %s") % dev) # device hasn't any partition elif deviceName == "": return True fdiskProg, gdiskProg = checkUtils('/sbin/fdisk', '/usr/sbin/gdisk') info = device.udev.get_device_info(name=dev) partitionNumber = (info.get('ID_PART_ENTRY_NUMBER','') or info.get('UDISKS_PARTITION_NUMBER', '')) devicePartitionCount = countPartitions(deviceName) if deviceName and not partitionNumber: raise DistributiveError( _("Failed to determine the partition number for %s") % dev) if parttable in ("dos", "primary", "extended", "logical"): fdisk = process(fdiskProg, deviceName, stderr=STDOUT) pipe = Popen([fdiskProg, deviceName], stdin=PIPE, stdout=PIPE, stderr=PIPE) if devicePartitionCount > 1: pipe.stdin.write("t\n%s\n%s\nw\n" % (partitionNumber, systemid)) else: pipe.stdin.write("t\n%s\nw\n" % systemid) pipe.stdin.close() pipe.wait() elif parttable == "gpt": pipe = Popen([gdiskProg, deviceName], stdin=PIPE, stdout=PIPE, stderr=PIPE) if devicePartitionCount > 1: pipe.stdin.write("t\n%s\n%s\nw\ny\n" % (partitionNumber, systemid)) else: pipe.stdin.write("t\n%s\nw\ny\n" % systemid) pipe.stdin.close() pipe.wait() for waittime in (0.1, 0.2, 0.5, 1, 2, 4): if path.exists(dev): return True else: sleep(waittime) raise DistributiveError( _( "Failed to found partition %s after changing the system ID") % dev) def formatSwapPartition(self, dev): """Format swap partition""" with open("/proc/swaps") as f: if dev in map(lambda y: y.split(" ")[0], filter(lambda x: x.startswith("/"),f)): raise DistributiveError( _("Failed to execute 'mkswap %s': " "the swap partition is used " "by the current system") % dev) if isMount(dev): raise DistributiveError( _("Failed to format %s: this partition is mounted") % dev) format_process = self.format_map["swap"](dev) if format_process.failed(): raise DistributiveError( _("Failed to format the swap partition") + " %s:\n%s" % (dev, format_process.readerr())) def installFrom(self, source, **kwargs): """Install distributive to partition from source distributive""" # get currect partition as directory distrTo = self.convertToDirectory() # install into directroy distributive from source distrTo.installFrom(source, **kwargs) def serialize(self): d = {'type': Distributive.Type.Partition, 'partition': self.partition, 'mdirectory': self.mdirectory, 'flag_remove_directory': self.flagRemoveDir, 'childs': [x.serialize() for x in self.childs]} return d @classmethod @Distributive.required("partition", "mdirectory", "flag_remove_directory") def unserialize(cls, data, parent=None): ld = PartitionDistributive(_u8(data['partition']), parent=parent) ld.mdirectory = _u8(data['mdirectory']) ld.flagRemoveDir = _u8(data['flag_remove_directory']) ld.childs = [Distributive.unserialize(x, parent=ld) for x in data.get('childs', [])] return ld class ArchiveDistributive(Distributive): def __init__(self, file, parent=None, mdirectory="/var/calculate/tmp/stage"): Distributive.__init__(self, parent=parent) self.file = file self.mdirectory = mdirectory def getType(self): return _("archive %s") % self.file def _detectArchive(self, file): """Detect archive by "/usr/bin/file" command Return bzip2,gzip,7z or None """ file_cmd = getProgPath('/usr/bin/file') p_file = process(file_cmd, file, stderr=process.STDOUT) if "bzip2 compressed data" in p_file.read(): return "bzip2" elif "gzip compressed data" in p_file.read(): return "gzip" elif "7-zip archive data" in p_file.read(): return "7z" elif "XZ compressed data" in p_file.read(): return "xz" elif file and file.endswith(".tar.lzma"): if path.exists('/usr/bin/7za'): return "7z" else: return "lzma" return None def _unpackArchive(self, archfile, directory): """Unpack archive""" # archive is exists if not path.exists(archfile): raise DistributiveError(_("File '%s' not found") % archfile) # detect type archive archiveType = self._detectArchive(archfile) # make directory if archive was detected normally if archiveType: self._makeDirectory(directory) # unpack archive tar_cmd = getProgPath('/bin/tar') if archiveType == "7z": arch_cmd = getProgPath('/usr/bin/7za') params = ["x", "-so"] elif archiveType == "lzma": arch_cmd = getProgPath('/usr/bin/lzma') params = ["-dc"] elif archiveType == "bzip2": arch_cmd = getProgPath('/bin/bunzip2') params = ["-dc"] elif archiveType == "gzip": arch_cmd = getProgPath('/bin/gzip') params = ["-dc"] elif archiveType == "xz": arch_cmd = getProgPath('/bin/xz') params = ["-dc"] else: raise DistributiveError(_("Unknown archive type '%s'") % archfile) if not arch_cmd or not tar_cmd: raise DistributiveError(_("Archive type '%s' is not supported") % archiveType) arch_process = process(*([arch_cmd] + params + [archfile])) tar_process = process(tar_cmd, "xf", "-", "-C", "%s/" % directory, stdin=arch_process) if tar_process.failed(): arch_error = arch_process.readerr().strip() tar_error = tar_process.readerr().strip() if arch_error: message = "%s\n%s" % (arch_error, tar_error) else: message = tar_error raise DistributiveError(_("Unpacking error\n%s") % message) def unpackTo(self, directory): """Unpack currect archive to directory""" self._unpackArchive(self.file, directory) def convertToDirectory(self): """Get archive as directory (unpack to directory)""" # check may be the archive already unpacked raise DistributiveError(_("Unsupported")) mdirectory = self.mdirectory for child in self.childs: if isinstance(child, DirectoryDistributive) and \ mdirectory in child.directory: return child # get temporary directory for unpacking mdirectory = self._getMntDirectory(mdirectory) dirdist = DirectoryDistributive(mdirectory, parent=self) self._unpackArchive(self.file, mdirectory) return dirdist def releaseChild(self, child): """Remove child Directory distributive""" if isinstance(child, DirectoryDistributive): self._removeDirectory(child.directory) child.directory = None def packToArchive(self, directory, file): tar_command = getProgPath("/usr/bin/tar") p = process(tar_command, "cf", file, "-C", directory, ".", stderr=process.STDOUT) if p.failed(): raise DistributiveError(_("Failed to create the archive") + " '%s':\n%s" % (file, p.read())) def installFrom(self, source, **kwargs): """Install distributive to partition from source distributive""" # get source distributive as directory distributive dFrom = source.convertToDirectory() # install into directroy distributive from source self.packToArchive(dFrom.directory, self.file) def serialize(self): d = {'type': Distributive.Type.Archive, 'file': self.file, 'mdirectory': self.mdirectory, 'childs': [x.serialize() for x in self.childs]} return d @classmethod @Distributive.required("mdirectory", "file") def unserialize(cls, data, parent=None): ld = ArchiveDistributive(_u8(data['file']), mdirectory=_u8(data['mdirectory']), parent=parent) ld.childs = [Distributive.unserialize(x, parent=ld) for x in data.get('childs', [])] return ld class ContainerDistributive(ArchiveDistributive): def __init__(self, basedir, parent=None, exclude=None, include=None, mdirectory="/var/calculate/tmp/stage"): Distributive.__init__(self, parent=parent) self.basedirectory = basedir self.file = path.join(basedir, "rootfs.tar.xz") self.meta = path.join(basedir, "meta.tar.xz") self.lxd = path.join(basedir, "lxd.tar.xz") self.exclude = [] if not exclude else exclude self.include = [] if not include else include self.mdirectory = mdirectory def mtime2build(self, fn): if path.exists(fn): build_time = datetime.datetime.fromtimestamp(os.stat(fn).st_mtime) return build_time.strftime("%Y%m%d") return "" def packToArchive(self, directory, file): if not path.exists(self.basedirectory): self._makeDirectory(self.basedirectory) tar_command = getProgPath("/usr/bin/tar") params = ["cf", file, "-J", "-C", directory] if self.exclude: exclude_list = list(calculate_exclude( directory, exclude=self.exclude, include=self.include)) params += list(map(lambda x: "--exclude=./%s" % x, exclude_list)) params += ["."] #debug_file = "/var/calculate/tmp/rootfs.tar.xz" #if path.exists(debug_file): # p = process("/bin/cp", debug_file, file) #else: p = process(tar_command, *params, stderr=process.STDOUT) try: if p.failed(): raise DistributiveError(_("Failed to create the archive") + " '%s':\n%s" % (file, p.read())) except BaseException: removeDir(self.basedirectory) raise def get_information(self): if path.exists(self.lxd): with xztaropen(self.lxd) as f: try: metadata_yaml = f.getmember("metadata.yaml") metadata = json.load(f.extractfile(metadata_yaml)) return { 'os_linux_build': metadata["calculate"]['os_linux_build'], 'os_arch_machine': metadata["calculate"]['os_arch_machine'], 'os_linux_shortname': metadata["calculate"]['os_linux_shortname'], 'os_linux_subname': metadata["calculate"]['os_linux_subname'], 'cl_profile_name': metadata["calculate"]['cl_profile_name'], 'os_linux_name': metadata["calculate"]['os_linux_name'], 'os_linux_ver': metadata["calculate"].get('os_linux_ver', "17"), } except (KeyError, ValueError) as e: pass return { 'os_linux_build': self.mtime2build(self.file), 'os_arch_machine': "x86_64", 'os_linux_shortname': 'Container', 'os_linux_subname': "", 'cl_profile_name': "", 'os_linux_name': _('Unknown Container') } class SquashDistributive(Distributive): def __init__(self, file, parent=None, mdirectory=None, exclude=None, compress="", include=None): Distributive.__init__(self, parent=parent) self.file = file self.mdirectory = mdirectory or DefaultMountPath.SquashImage self.exclude = exclude or [] self.include = include or [] self.compress = compress if compress and compress != "gzip" else "" def getType(self): return _("squash image %s") % self.file def _mountSquash(self, file, directory): """Mount squashfs to directory""" self._makeDirectory(directory) self._mountToDirectory(file, directory, mountopts="-o loop -t squashfs") def _umountSquash(self, directory): self._umountDirectory(directory) self._removeDirectory(directory) def convertToDirectory(self): mdirectory = self.mdirectory for child in self.childs: if isinstance(child, DirectoryDistributive) and \ child.directory and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._mountSquash(self.file, mdirectory) return DirectoryDistributive(mdirectory, parent=self) def releaseChild(self, child): """Umount child Directory distributive""" if isinstance(child, DirectoryDistributive): self._umountSquash(child.directory) child.directory = None def packToSquash(self, directory, file, **kwargs): mksquashfsUtil = '/usr/bin/mksquashfs' if not path.exists(mksquashfsUtil): raise DistributiveError( _("Failed to create squash") + " : %s" % _("command '%s' not found") % mksquashfsUtil) cmd = [mksquashfsUtil, "%s/" % directory, file] if self.compress: cmd += ["-comp", self.compress] cmd += ["-progress"] if self.exclude: exclude_list = list(calculate_exclude(directory, exclude=self.exclude, include=self.include)) cmd += ["-e"] + exclude_list # возможность использовать заранее подготовленный livecd.squashfs if path.exists('/var/calculate/tmp/livecd.squashfs'): os.system('cp -L /var/calculate/tmp/livecd.squashfs %s' % file) else: callbackProgress = kwargs.get('callbackProgress', None) processMkSquash = PercentProgress(*cmd, stderr=STDOUT, atty=True) for perc in processMkSquash.progress(): if callable(callbackProgress): callbackProgress(perc) if processMkSquash.failed(): raise DistributiveError(_("Failed to create squashfs") + " '%s':\n%s" % ( file, processMkSquash.read())) def installFrom(self, source, **kwargs): """Install distributive to partition from source distributive""" # get source distributive as directory distributive dFrom = source.convertToDirectory() # install into directroy distributive from source self.packToSquash(dFrom.directory, self.file, **kwargs) def serialize(self): d = {'type': Distributive.Type.SquashFS, 'file': self.file, 'mdirectory': self.mdirectory, 'childs': [x.serialize() for x in self.childs]} return d @classmethod @Distributive.required("mdirectory", "file") def unserialize(cls, data, parent=None): ld = SquashDistributive(_u8(data['file']), mdirectory=_u8(data['mdirectory']), parent=parent) ld.childs = [Distributive.unserialize(x, parent=ld) for x in data.get('childs', [])] return ld class IsoDistributive(Distributive): def __init__(self, file, parent=None, mdirectory=None, bdirectory=None, exclude=None, compress="gzip", vol_id="CALCULATE", include=None): if bdirectory is None: bdirectory = DefaultMountPath.BuildDirectory Distributive.__init__(self, parent=parent) self.file = file self.vol_id = vol_id if path.isdir(self.file): self.mdirectory = self.file else: self.mdirectory = mdirectory or DefaultMountPath.IsoImage if file == bdirectory: self.bdirectory = file else: self.bdirectory = self._getMntDirectory(bdirectory) self.exclude = [] if not exclude else exclude self.include = [] if not include else include self.compress = compress self.eventPrepareIso = Signal() def getType(self): tf = typeFile(magic=MAGIC_COMPRESS | MAGIC_SYMLINK | MAGIC_CONTINUE) ftype = tf.getMType(self.file) if "block special" in ftype: return _("live image %s") % self.file if path.isdir(self.file): return _("image directory %s") % self.file else: return _("ISO image %s") % self.file def probe(self): """Check directory for iso content""" try: pathname = self.getIsoContentDirectory() except Exception: return False return path.exists(path.join(pathname, "syslinux")) and \ path.exists(path.join(pathname, "isolinux")) def get_squash_size(self): try: dn = self.getIsoContentDirectory() fn = path.join(dn, "livecd.squashfs") return path.getsize(fn) except Exception: raise DistributiveError(_("Failed to get size of the squash image")) def _mountIso(self, file, directory): if self.file != self.mdirectory: self._makeDirectory(directory) tf = typeFile(magic=MAGIC_COMPRESS | MAGIC_SYMLINK | MAGIC_CONTINUE) ftype = tf.getMType(file) if "block special" in ftype: mopts = "-o ro" else: mopts = "-o ro,loop" self._mountToDirectory(file, directory, mountopts=mopts) def _umountIso(self, directory): if self.file != self.mdirectory: self._umountDirectory(directory) self._removeDirectory(directory) def convertToSquash(self): mdirectory = self.mdirectory for child in self.childs: if isinstance(child, SquashDistributive) and \ mdirectory in child.file: return child if self.mdirectory != self.file: mdirectory = self._getMntDirectory(mdirectory) self._mountIso(self.file, mdirectory) fileLive = self._getLastLive(mdirectory) if not fileLive: self._umountIso(mdirectory) raise DistributiveError(_("ISO %s contains no live image") % self.file) return SquashDistributive(path.join(mdirectory, fileLive), parent=self, exclude=self.exclude, compress=self.compress, include=self.include) def getIsoContentDirectory(self): """Return directory with content of iso image""" squash = self.convertToSquash() return path.dirname(squash.file) def releaseChild(self, child): """Umount child Directory distributive""" if isinstance(child, SquashDistributive): self._umountIso(path.dirname(child.file)) child.directory = None def convertToDirectory(self): return self.convertToSquash().convertToDirectory() def _get_iso_util(self): mkisoUtil = '/usr/bin/mkisofs' if not path.exists(mkisoUtil): raise DistributiveError( "{errmess} : {errdescr}".format( errmess=_("Failed to create the ISO image"), errdescr=_("command '%s' not found") % mkisoUtil)) return mkisoUtil def getMkIso(self, output=None, source=None, volume_id=None, efi_image=None): """ Параметры mkisofs при создании образа с поддержкой EFI и обычного обарза :param output: :param source: :param volume_id: :param efi_image: :return: """ mkisoUtil = self._get_iso_util() if efi_image is None: params = ["-b", "isolinux/isolinux.bin", "-no-emul-boot", "-V", volume_id, "-boot-load-size", "4", "-boot-info-table", "-iso-level", "4", "-hide", "boot.catalog"] else: params = ["-J", "-R", "-l", "-no-emul-boot", "-boot-load-size", "4", "-udf", "-boot-info-table", "-iso-level", "4", "-b", "isolinux/isolinux.bin", "-V", volume_id, "-c", "isolinux/boot.cat", "-eltorito-alt-boot", "-no-emul-boot", "-eltorito-platform", "efi", "-eltorito-boot", efi_image] return [mkisoUtil] + params + ["-o", output, "%s/" % source] def packToIso(self, directory, file, **kwargs): # remove previous version of iso try: if path.exists(file): os.unlink(file) except (Exception, KeyboardInterrupt) as e: raise DistributiveError(_("Failed to remove") + " %s:\n%s" % (file, str(e))) if not path.exists(path.dirname(file)): makeDirectory(path.dirname(file)) efi_image = 'boot/grub/efi.img' if path.exists(path.join(directory, efi_image)): cmd = self.getMkIso(source=directory, output=file, volume_id=self.vol_id, efi_image=efi_image) else: cmd = self.getMkIso(source=directory, output=file, volume_id=self.vol_id) callback_progress = kwargs.get('callbackProgress', None) processMkIsoFs = PercentProgress(*cmd, stderr=STDOUT, atty=True) for perc in processMkIsoFs.progress(): if callable(callback_progress): callback_progress(perc) if processMkIsoFs.failed(): raise DistributiveError(_("Failed to create the ISO image") + " %s:\n%s" % (file, processMkIsoFs.read())) else: return True def installFrom(self, source, **kwargs): """Install distributive to partition from source distributive""" # make temporary directory for creating iso image isoDirectory = self.bdirectory self._makeDirectory(isoDirectory) try: # getting squash from source liveimage = self._getLastLive(isoDirectory) if liveimage: curnum = self._getSquashNum(self.reLive.search(liveimage)) liveimage = "livecd.squashfs.%d" % (curnum + 1) else: liveimage = "livecd.squashfs" if isinstance(source, SquashDistributive): self._copyfile(source.file, path.join(isoDirectory, liveimage)) else: distDirectory = source.convertToDirectory() squashDistr = SquashDistributive( path.join(isoDirectory, liveimage), exclude=self.exclude, include=self.include, compress=self.compress) squashDistr.installFrom(distDirectory, **kwargs) # prepare iso self.eventPrepareIso.emit(isoDirectory) # pack iso if self.bdirectory != self.file: self.packToIso(isoDirectory, self.file, **kwargs) except DistributiveError as e: raise e except KeyboardInterrupt as e: raise DistributiveError(_("Keyboard interruption")) def close(self): # close all child if Distributive.close(self): # remove temporary directory if path.lexists(self.bdirectory) and self.file != self.bdirectory: self._removeDirectory(self.bdirectory) def serialize(self): d = {'type': Distributive.Type.Iso, 'file': self.file, 'mdirectory': self.mdirectory, 'bdirectory': self.bdirectory, 'childs': [x.serialize() for x in self.childs]} return d @classmethod @Distributive.required("mdirectory", "bdirectory", "file") def unserialize(cls, data, parent=None): ld = IsoDistributive(_u8(data['file']), mdirectory=_u8(data['mdirectory']), bdirectory=_u8(data['bdirectory']), parent=parent) ld.childs = [Distributive.unserialize(x, parent=ld) for x in data.get('childs', [])] return ld class FlashDistributive(PartitionDistributive): def _checkMount(self, dev): """Checking mount point""" mp = isMount(dev) if mp: if mp.startswith('/media'): self._umountDirectory(mp) else: raise DistributiveError( _("Failed to format %s: this partition is mounted") % dev) def performFormat(self): """Perform format for all partition of this distributive""" if not self.isFormat: dn = self.getDirectory() clear_match = re.compile( r"^(boot|efi|isolinux|syslinux|id.*\.uefi|" "ldlinux.c32|ldlinux.sys|livecd|livecd.squashfs)$") for fn in list(filter(clear_match.match, listDirectory(dn))): full_path = path.join(dn, fn) try: if path.isdir(full_path): removeDir(full_path) else: os.unlink(full_path) except (OSError, IOError): raise DistributiveError( _("Failed to remove %s") % fn) else: super(FlashDistributive, self).performFormat() def getType(self): return _("USB flash %s") % self.partition def probe(self): """Check directory for flash content""" try: pathname = self.getDirectory() except Exception: return False return path.exists(path.join(pathname, "syslinux")) and \ path.exists(path.join(pathname, "isolinux")) def writeable(self, mp): """ Перемонитровать в rw, при необходимости """ if not check_rw(mp): # перемонитровать в rw p = process('/bin/mount', '-o', 'remount,rw', mp, stderr=STDOUT) if p.success() and check_rw(mp): return True return False else: return True def convertToDirectory(self): mp = isMount(self.partition) if mp and self.writeable(mp): d = DirectoryDistributive(mp) d.no_unmount = True return d return super(FlashDistributive, self).convertToDirectory() def releaseChild(self, child): """Umount child Directory distributive""" if isinstance(child, DirectoryDistributive): if not child.no_unmount: self._umountPartition(child.directory) child.directory = None def installFrom(self, source, **kwargs): """Install distributive to partition from source distributive""" # make temporary directory for creating iso image distrTo = self.convertToDirectory() # getting squash from source if isinstance(source, IsoDistributive): self.rsync(source.getIsoContentDirectory(), distrTo.directory, byfile=lambda x: x.startswith('livecd.'), noperm=True, **kwargs) else: raise DistributiveError( _("Flash install does not support %s") % source.__class__.__name__) class PxeDistributive(Distributive): needFormat = False def __init__(self, directory, parent=None): Distributive.__init__(self, parent=parent) self.directory = path.join(directory, "calculate") self.origdir = directory def getDirectory(self): return self.origdir def installFrom(self, source, **kwargs): # make temporary directory for creating iso image distrTo = self.directory # getting squash from source if isinstance(source, IsoDistributive): if path.exists(self.directory): removeDir(self.directory) self._makeDirectory(self.directory) self.rsync(source.getIsoContentDirectory(), distrTo, byfile=lambda x: x.startswith('livecd.'), **kwargs) else: raise DistributiveError( _("PXE install does not support %s" % source.__class__.__name__)) class LayeredDistributive(Distributive): """ Каталог дистрибутива для сборки """ needFormat = False def __init__(self, mdirectory, diff_directory, image_file=None, parent=None): """ :param mdirectory: результирующий каталог :param diff_directory: каталог содержит изменения от оригинала :param image_file: образ оригинала :param parent: родительский дистрибутив """ super(LayeredDistributive, self).__init__(parent=parent) self.mdirectory = mdirectory self.diff_directory = diff_directory self.workdir = "%s-work" % self.diff_directory self.image_mount_dir = None self.image_distro = None self.image_file = image_file def post_clear(self): if path.exists(self.diff_directory): self._removeDirectory(self.diff_directory) self.clear_empty_directories(self.diff_directory) return True def getType(self): return _("layered '{file} {diff}'").format( file=self.image_file, diff=self.diff_directory) def clearDiff(self): if path.exists(self.diff_directory): self._removeDirectory(self.diff_directory) def _mountLayers(self, target): """Mount squashfs to directory""" self._makeDirectory(target) if not path.exists(self.diff_directory): self._makeDirectory(self.diff_directory) if path.exists(self.workdir): self._removeDirectory(self.workdir) if not path.exists(self.workdir): self._makeDirectory(self.workdir) self._mountToDirectory("none", target, mountopts=( "-t overlay -o lowerdir=%(static)s," "upperdir=%(upper)s,workdir=%(workdir)s" % {"upper": self.diff_directory, "workdir": self.workdir, "static": self.image_mount_dir})) def _umountLayers(self, directory): self._umountDirectory(directory) self._removeDirectory(directory) self._removeDirectory(self.workdir) def _mountLiveImage(self): """Mount squashfs to directory""" self.image_distro = IsoDistributive(file=self.image_file, parent=self) self.image_mount_dir = ( self.image_distro.convertToDirectory().getDirectory()) def _umountLiveImage(self): if self.image_distro: self.image_distro.close() self.image_distro = None def convertToDirectory(self): """Convert scrach directories to one directory""" mdirectory = self.mdirectory for child in self.childs: if isinstance(child, DirectoryDistributive) and \ mdirectory in child.directory: return child mdirectory = self._getMntDirectory(mdirectory) self._mountLiveImage() self._mountLayers(mdirectory) return DirectoryDistributive(mdirectory, parent=self) def releaseChild(self, child): """Unmount child Directory distributive""" if isinstance(child, DirectoryDistributive): self._umountLayers(child.directory) self._umountLiveImage() child.directory = None def installFrom(self, source, **kwargs): """Install distributive to partition from source distributive""" # make temporary directory for creating iso image if isinstance(source, (IsoDistributive, SquashDistributive)): self.image_file = source.file return True raise DistributiveError( _("Install with layers does not support %s") % source.__class__.__name__) def serialize(self): d = {'type': Distributive.Type.Layered, 'mdirectory': self.mdirectory, 'diff_directory': self.diff_directory, 'image_file': self.image_file, 'childs': [x.serialize() for x in self.childs]} if self.image_mount_dir: d['image_mount_dir'] = self.image_mount_dir return d @classmethod @Distributive.required("mdirectory", "diff_directory", "image_file") def unserialize(cls, data, parent=None): ld = LayeredDistributive(_u8(data['mdirectory']), _u8(data['diff_directory']), _u8(data['image_file']), parent=parent) ld.childs = [Distributive.unserialize(x, parent=ld) for x in data.get('childs', [])] return ld