From ab4c13a46a7a85717ab1dba1d0e89463ab400c93 Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Wed, 4 Oct 2017 14:01:57 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BF=D0=B8=D1=81?= =?UTF-8?q?=D0=B0=D0=BD=D0=B0=20=D0=B0=D0=B2=D1=82=D0=BE=D1=80=D0=B0=D0=B7?= =?UTF-8?q?=D0=BC=D0=B5=D1=82=D0=BA=D0=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Модуль install теперь используется utils/partition --- pym/calculate/lib/datavars.py | 1 + pym/calculate/lib/utils/debug.py | 54 +- pym/calculate/lib/utils/device.py | 222 ++++-- pym/calculate/lib/utils/partition.py | 1088 ++++++++++++++++++++++++++ pym/calculate/lib/utils/tools.py | 9 +- 5 files changed, 1316 insertions(+), 58 deletions(-) create mode 100644 pym/calculate/lib/utils/partition.py diff --git a/pym/calculate/lib/datavars.py b/pym/calculate/lib/datavars.py index 0d05632..33d686e 100644 --- a/pym/calculate/lib/datavars.py +++ b/pym/calculate/lib/datavars.py @@ -1099,6 +1099,7 @@ class SimpleDataVars(object): 'ne': lambda x, y: lambda z: z[x] != y, 'in': lambda x, y: lambda z: z[x] in y, 'not_in': lambda x, y: lambda z: z[x] not in y, + 'startswith': lambda x, y: lambda z: z[x].startswith(y) } filter_chain.append(func_maps[funcname](len(variables) - 1, kw[k])) l = len(fields) diff --git a/pym/calculate/lib/utils/debug.py b/pym/calculate/lib/utils/debug.py index 6fb2a61..c93bbc2 100644 --- a/pym/calculate/lib/utils/debug.py +++ b/pym/calculate/lib/utils/debug.py @@ -15,7 +15,8 @@ # limitations under the License. from calculate.lib.utils.device import (Sysfs, UdevAdm, sysfs, Devfs, Udev, Lvm, - UdevAdmNull) + UdevAdmNull, LvmCommand, MdadmCommand, + Raid) from calculate.lib.utils.files import writeFile from calculate.lib.utils.vfs import (VFSKeeper, VFSSysfsImporter, SafeVFS, @@ -71,25 +72,55 @@ class VirtualUdevAdm(UdevAdmNull): pass -class VirtualLvm(Lvm): +class VirtualLvmCommand(object): def __init__(self, output): - super(VirtualLvm, self).__init__() self.output = output - def pvdisplay_output(self, output): - if output == "vg_name,lv_name,pv_name": - return self.output + def get_physical_extent_size(self): + return "4096" - def refresh(self): + def get_pvdisplay_output(self, output): + return self.output + + def vgscan(self): + return True + + def vgchange(self): + return True + + def lvchange(self, groups): + return True + + def execute(self, *command): + return True + + def double_execute(self, *command): + return True + + def remove_lv(self, vg, lv): + return True + + def remove_vg(self, vg): + return True + + def remove_pv(self, pv): + return True + + +class VirtualMdadmCommand(object): + def zero_superblock(self, devices): pass + def stop_raid(self, devraid): + pass class VirtualDeviceModule(object): def __init__(self): self.sysfs = Sysfs() self.devfs = Devfs() self.udev = Udev() - self.lvm = Lvm() + self.lvm = Lvm(LvmCommand()) + self.raid = Raid(MdadmCommand()) def set_sysfs(self, f): self.sysfs = Sysfs(SafeVFS(VFSKeeper.load(f))) @@ -100,11 +131,14 @@ class VirtualDeviceModule(object): def set_udev(self, f): self.udev.udevadm = VirtualUdevAdm(f.read(), self) + def set_raid(self): + self.raid = Raid(VirtualMdadmCommand()) + def set_lvm(self, f): if f is None: - self.lvm = VirtualLvm("") + self.lvm = Lvm(VirtualLvmCommand("")) else: - self.lvm = VirtualLvm(f.read()) + self.lvm = Lvm(VirtualLvmCommand(f.read())) class DumpError(Exception): pass diff --git a/pym/calculate/lib/utils/device.py b/pym/calculate/lib/utils/device.py index ceeafff..3eb2592 100644 --- a/pym/calculate/lib/utils/device.py +++ b/pym/calculate/lib/utils/device.py @@ -18,8 +18,9 @@ import sys import re import os from os import path -from calculate.lib.utils.tools import (Cachable, GenericFs, sorteduniqresult) +from calculate.lib.utils.tools import (Cachable, GenericFs, unique) import files +from time import sleep from calculate.lib.cl_lang import setLocalTranslate @@ -60,27 +61,89 @@ def countPartitions(devname): return len([x for x in sysfs.listdir(syspath) if x.startswith(device_name)]) +class LvmCommand(object): + @property + def lvm_cmd(self): + return files.getProgPath('/sbin/lvm') + + def get_physical_extent_size(self): + if not self.lvm_cmd: + return "" + pvdata = files.process(self.lvm_cmd, "lvmconfig", "--type", "full", + "allocation/physical_extent_size") + if pvdata.success(): + return pvdata.read().strip().rpartition("=")[-1] + return "" + + def get_pvdisplay_output(self, output): + if not self.lvm_cmd: + return "" + pvdata = files.process(self.lvm_cmd, "pvdisplay", "-C", "-o", + output, "--noh") + if pvdata.success(): + return pvdata.read() + return "" + + def vgscan(self): + if self.lvm_cmd: + return files.process(self.lvm_cmd, "vgscan").success() + return False + + def vgchange(self): + failed = True + if self.lvm_cmd: + failed = files.process("vgchange", '-ay').failed() + failed |= files.process("vgchange", '--refresh').failed() + return not failed + + def lvchange(self, groups): + failed = True + if self.lvm_cmd: + failed = False + for group in groups: + failed |= files.process("lvchange", '-ay', group).failed() + failed |= files.process("lvchange", '--refresh', group).failed() + return not failed + + def execute(self, *command): + if self.lvm_cmd: + return files.process(self.lvm_cmd, *command).success() + return False + + def double_execute(self, *command): + if not self.execute(*command): + sleep(2) + return self.execute(*command) + return False + + def remove_lv(self, vg, lv): + return self.double_execute("lvremove", "%s/%s" % (vg, lv), "-f") + + def remove_vg(self, vg): + return self.double_execute("vgremove", vg, "-f") + + def remove_pv(self, pv): + return self.double_execute("pvremove", pv, "-ffy") + class Lvm(Cachable): """ LVM информация """ - @property - def pvdisplay_cmd(self): - return files.getProgPath('/sbin/pvdisplay') + def __init__(self, commander): + super(Lvm, self).__init__() + self.commander = commander @property def groups(self): return sorted({vg for vg, lv, pv in self.pvdisplay_full()}) + @Cachable.methodcached() + def get_pesize(self): + return self.commander.get_physical_extent_size() + @Cachable.methodcached() def pvdisplay_output(self, output): - if not self.pvdisplay_cmd: - return "" - pvdata = files.process(self.pvdisplay_cmd, "-C", "-o", - output, "--noh") - if pvdata.success(): - return pvdata.read() - return "" + return self.commander.get_pvdisplay_output(output) def pvdisplay_full(self): data = self.pvdisplay_output("vg_name,lv_name,pv_name").strip() @@ -96,18 +159,19 @@ class Lvm(Cachable): def refresh(self): """Run command which refresh information about LVM""" - vgscan = files.getProgPath('/sbin/vgscan') - vgchange = files.getProgPath('/sbin/vgchange') - lvchange = files.getProgPath('/sbin/lvchange') + if not os.environ.get('EBUILD_PHASE'): + self.commander.vgscan() + self.commander.vgchange() + self.commander.lvchange(lvm.groups) - if vgscan and vgchange and lvchange: - files.process(vgscan).success() - files.process(vgchange, '-ay').success() - files.process(vgchange, '--refresh').success() - for group in lvm.groups: - if not os.environ.get('EBUILD_PHASE'): - files.process(lvchange, '-ay', group).success() - files.process(lvchange, '--refresh', group).success() + def remove_lv(self, vg, lv): + return self.commander.remove_lv(vg, lv) + + def remove_vg(self, vg): + return self.commander.remove_vg(vg) + + def remove_pv(self, pv): + return self.commander.remove_pv(pv) def lspci(filtername=None, shortInfo=False): @@ -259,14 +323,22 @@ class Udev(object): """ Получить все устройства SUBSYSTEM=block устройства """ - block_entries = (x for x in self.udevadm.info_export().split("\n\n") - if "SUBSYSTEM=block" in x) - for block in block_entries: - for entry in (x[3:] for x in block.split("\n") - if x.startswith("P:")): - if "/block/" in entry: - yield entry[entry.rindex("/block/"):] - break + for block in sysfs.glob(sysfs.Path.Block, "*"): + yield block + blockname = path.basename(block) + for part in sysfs.glob(block, "%s*" % blockname): + yield part + + #block_devices = {sysfs.realpath(x)[len(sysfs.base_dn):]: x + # for x in sysfs.listdir("/block/*", fullpath=True)} + #block_entries = (x for x in self.udevadm.info_export().split("\n\n") + # if "SUBSYSTEM=block" in x) + #for block in block_entries: + # for entry in (x[3:] for x in block.split("\n") + # if x.startswith("P:")): + # if "/block/" in entry: + # yield entry[entry.rindex("/block/"):] + # break def syspath_to_devname(self, it, dropempty=True): """ @@ -371,33 +443,63 @@ class Udev(object): return "primary" return info.get('ID_PART_TABLE_TYPE', '') - @sorteduniqresult - def get_disk_devices(self, path=None, name=None): - """Get real parent device by partition,lvm,mdraid""" + def _get_disk_devices(self, path=None, name=None): + """ + Возвращает список базовых устройств (признак базовое ли устройство, + название устройства) + :param path: + :param name: + :return: + """ info = udev.get_device_info(path=path, name=name) syspath = info.get("DEVPATH", '') if syspath: # real device if self.is_device(info): - yield info.get("DEVNAME", "") + yield True, info.get("DEVNAME", "") # partition elif self.is_partition(info): - for x in self.get_disk_devices(path=os.path.dirname(syspath)): + for x in self._get_disk_devices(path=os.path.dirname(syspath)): yield x # md raid elif udev.is_raid(info): - for rd in raid.devices_syspath(syspath): - for x in self.get_disk_devices(path=rd): + yield False, info.get("DEVNAME", "") + for rd in sorted(raid.devices_syspath(syspath)): + for x in self._get_disk_devices(path=rd): yield x # lvm elif udev.is_lvm(info): + yield False, info.get("DEVNAME", "") for lvdev in lvm.used_partitions(info["DM_VG_NAME"], info["DM_LV_NAME"]): - for x in self.get_disk_devices(name=lvdev): + for x in self._get_disk_devices(name=lvdev): yield x + def get_disk_devices(self, path=None, name=None): + """Get real parent device by partition,lvm,mdraid""" + return sorted({ + dev + for realdevice, dev in self._get_disk_devices(path, name) + if realdevice + }) -def humanreadableSize(size): + def get_all_base_devices(self, path=None, name=None): + """ + Получить все устройства (включая RAID и LVM) из которого состоит + указанное устройство, исключая loop устройства + """ + try: + devs = (dev + for realdevice, dev in self._get_disk_devices(path, name) + if not dev.startswith("/dev/loop")) + if not self.is_partition(self.get_device_info(path, name)): + next(devs) + return list(unique(devs)) + except StopIteration: + return [] + + +def humanreadableSize(size, compsize=0): """ Human readable size (1024 -> 1K and etc) """ @@ -411,7 +513,7 @@ def humanreadableSize(size): ((1024 ** 3), "G", True), ((1024 ** 4), "T", True), ((1024 ** 5), "P", True)) - suffix = filter(lambda x: size > x[0], suffix) + suffix = filter(lambda x: abs(size-compsize) > x[0], suffix) if suffix: suffix = suffix[-1] printSize = int(size / (float(suffix[0]) / 10)) @@ -442,14 +544,32 @@ def getPartitionSize(syspath=None, name=None, inBytes=False): return "" +class MdadmCommand(object): + @property + def mdadm_cmd(self): + return files.getProgPath('/sbin/mdadm') + + def stop_raid(self, devraid): + if not self.mdadm_cmd: + return "" + pvdata = files.process(self.mdadm_cmd, "-S", devraid) + return pvdata.success() + + def zero_superblock(self, devices): + if not self.mdadm_cmd: + return "" + for dev in devices: + files.process(self.mdadm_cmd, "--zero-superblock", dev).success() + class Raid(object): + def __init__(self, commander): + self.commander = commander + def devices_info(self, raid): prop = udev.get_device_info(path=raid) if udev.is_raid(prop): - for rd in (x for x in sysfs.listdir(raid, "md", fullpath=True) - if path.basename(x).startswith('rd')): - if sysfs.exists(rd, "block"): - yield udev.get_device_info(sysfs.syspath(rd, "block")) + for rdblock in sysfs.glob(raid, "md/dev-*", "block"): + yield udev.get_device_info(sysfs.syspath(rdblock)) def devices_syspath(self, raid): """ @@ -462,7 +582,8 @@ class Raid(object): def devices(self, raid, pathname="DEVNAME"): """ - Получить устройства /dev + Получить устройства /dev из которых состоит RAID, не возвращает + список этих устройств для раздела сделанного для RAID (/dev/md0p1) :param raid: :param pathname: :return: @@ -472,6 +593,13 @@ class Raid(object): if devname: yield devname + def remove_raid(self, raidname): + raidparts = self.devices(udev.get_syspath(name=raidname)) + failed = False + self.commander.stop_raid(raidname) + for dev in raidparts: + self.commander.zero_superblock(dev) + return failed def loadEfiVars(): """ @@ -576,5 +704,5 @@ class Devfs(DeviceFs): sysfs = Sysfs() devfs = Devfs() udev = Udev() -lvm = Lvm() -raid = Raid() +lvm = Lvm(LvmCommand()) +raid = Raid(MdadmCommand()) diff --git a/pym/calculate/lib/utils/partition.py b/pym/calculate/lib/utils/partition.py new file mode 100644 index 0000000..334a2c2 --- /dev/null +++ b/pym/calculate/lib/utils/partition.py @@ -0,0 +1,1088 @@ +# -*- coding: utf-8 -*- + +# Copyright 2017 Mir Calculate. http://www.calculate-linux.org +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +from abc import ABCMeta, abstractmethod +from calculate.lib.utils.files import process, getProgPath +from calculate.lib.utils.tools import Sizes, classificate +from calculate.lib.utils.device import humanreadableSize + +_ = lambda x: x +from calculate.lib.cl_lang import setLocalTranslate + +setLocalTranslate('cl_lib3', sys.modules[__name__]) + +class VolumesError(Exception): + pass + +class VolumesBuilder(object): + + __metaclass__ = ABCMeta + + class DeviceType(object): + Device = 0 + RAID = 1 + + class VolumeType(object): + Generic = 0 + Raid = 2 + Fat32 = 3 + Linux = 4 + LinuxSwap = 5 + Ntfs = 6 + BiosBoot = 7 + Efi = 8 + Lvm = 9 + Extended = 10 + + class Purpose(object): + Undefined = 100 + EfiBoot = 101 + BiosBoot = 102 + MainRoot = 103 + UpdateRoot = 104 + Swap = 105 + Calculate = 106 + Lvm = 107 + Windows = 108 + Raid = 109 + Linux = 110 + + purpose_type = { + Purpose.Undefined: VolumeType.Generic, + Purpose.Lvm: VolumeType.Lvm, + Purpose.BiosBoot: VolumeType.BiosBoot, + Purpose.EfiBoot: VolumeType.Efi, + Purpose.MainRoot: VolumeType.Linux, + Purpose.UpdateRoot: VolumeType.Linux, + Purpose.Calculate: VolumeType.Linux, + Purpose.Linux: VolumeType.Linux, + Purpose.Swap: VolumeType.LinuxSwap, + Purpose.Windows: VolumeType.Ntfs, + Purpose.Raid: VolumeType.Raid + } + + SizeAllFree = "allfree" + + @abstractmethod + def create(self): + """ + Пересоздать таблицу разделов + """ + pass + + @abstractmethod + def create_volume(self, size, name, purpose=0): + pass + + @abstractmethod + def create_last_volume(self, size, name, purpose=0): + pass + + @abstractmethod + def set_minimal_volume_size(self, size): + pass + + @abstractmethod + def finish(self): + pass + + +class DosDisk(object): + __metaclass__ = ABCMeta + + @abstractmethod + def get_partition_name(self, num): + pass + + @abstractmethod + def create_table(self): + pass + + @abstractmethod + def create_primary_partition(self, size): + pass + + @abstractmethod + def create_logical_partition(self, size): + pass + + @abstractmethod + def create_extended_partition(self): + pass + + @abstractmethod + def change_partition_type(self, partid, partnum=None): + pass + + @abstractmethod + def write(self): + pass + + +class GptDisk(object): + __metaclass__ = ABCMeta + + @abstractmethod + def create_table(self): + pass + + @abstractmethod + def create_partition(self, size): + pass + + @abstractmethod + def change_partition_type(self, partid, partnum=None): + pass + + @abstractmethod + def write(self): + pass + + @abstractmethod + def get_partition_name(self, num): + pass + + +class LvmDisk(object): + __metaclass__ = ABCMeta + + @abstractmethod + def create_lvm(self): + pass + + @abstractmethod + def create_logical_volume(self, name, size): + pass + + @abstractmethod + def write(self): + pass + + +class BlockDisk(object): + pass + +class VirtualDiskError(Exception): + pass + +class SizeableDisk(object): + AllFree = None + header_size = 0 + + def __init__(self, size): + self.disk_size = size + self.free_size = size - self.header_size + + def occupe_size(self, size): + if size is not self.AllFree: + if size < 0: + self.free_size = size + else: + self.free_size -= size + else: + self.free_size = 0 + + def size(self, size): + if size is not self.AllFree: + if size < 0: + return self.free_size + size + else: + return size + else: + return self.free_size + + def check_free_size(self): + if self.free_size < 0: + needbytes = abs(self.free_size) + availbytes = self.disk_size + avail = humanreadableSize(availbytes) + need = humanreadableSize(needbytes) + raise VirtualDiskError( + _("There is not enough space on this device") + _(": ") + + _("{avail} available, need at {need} more").format( + avail=avail, + need=need)) + + +class VirtualDisk(SizeableDisk, DosDisk, GptDisk): + parttype = "" + + class Partition(object): + def __init__(self, dev=None, mount=None, part=None, + partid=None, size=None): + self.dev = dev + self.mount = mount + self.part = part + self.partid = partid + self.size = size + self.format = "" + + def __init__(self, parent, dev, size): + self.parent = parent + self.dev = dev + self.partitions = [] + super(VirtualDisk, self).__init__(size) + + volume_types_map = {} + + def get_partition_name(self, num): + if self.dev[-1].isdigit(): + return ("%sp%%d" % self.dev) % num + else: + return ("%s%%d" % self.dev) % num + + def create_table(self): + self.partitions = [] + + def create_partition(self, size, parttype=None): + real_size = self.size(size) + self.occupe_size(real_size) + self.partitions.append( + self.Partition( + dev=self.get_partition_name(len(self.partitions)+1), + mount="", part=parttype or self.parttype, size=real_size) + ) + + def create_primary_partition(self, size): + self.create_partition(size, parttype="primary") + + def create_logical_partition(self, size): + self.create_partition(size, parttype="logical") + + def create_extended_partition(self): + self.create_partition(0, parttype="extended") + self._get_partition().size = self.free_size + self._get_partition().partid = DosPartitionId.get( + VolumesBuilder.VolumeType.Extended) + + def _get_partition(self, partnum=None): + if partnum is None: + return self.partitions[-1] + else: + return self.partitions[partnum] + + def change_partition_type(self, partid, partnum=None): + part = self._get_partition(partnum) + part.partid = partid + + def change_purpose(self, purpose, partnum=None): + part = self._get_partition(partnum) + + part.mount = self.parent.get_mount_point(purpose) + if purpose == VolumesBuilder.Purpose.EfiBoot: + self.parent.set_efi(part.dev) + part.format = self.parent.get_format(purpose) + + if purpose is VolumesBuilder.Purpose.UpdateRoot: + self.parent.dev_from = part.dev + + def write(self): + self.check_free_size() + devtype = self.parent.get_device_type(self.dev) + for partition in self.partitions: + disk_type = "%s-partition" % devtype + if partition.partid in ( + GptPartitionId.get(VolumesBuilder.VolumeType.BiosBoot), + DosPartitionId.get(VolumesBuilder.VolumeType.Extended), + DosPartitionId.get(VolumesBuilder.VolumeType.Lvm), + GptPartitionId.get(VolumesBuilder.VolumeType.Lvm), + ): + if partition.partid == GptPartitionId.get( + VolumesBuilder.VolumeType.BiosBoot): + self.parent.set_mbr(self.dev) + elif partition.partid in ( + DosPartitionId.get(VolumesBuilder.VolumeType.Lvm), + GptPartitionId.get(VolumesBuilder.VolumeType.Lvm)): + self.parent.lvm_size[partition.dev] = partition.size + self.parent.lvm_disk_type[partition.dev] = disk_type + continue + self.parent.disk_dev.append(partition.dev) + self.parent.disk_mount.append(partition.mount) + self.parent.disk_part.append(partition.part) + self.parent.disk_size.append(partition.size) + self.parent.disk_type.append(disk_type) + self.parent.disk_format.append(partition.format or "ext4") + + +class VirtualDosDisk(VirtualDisk, DosDisk): + parttype = "dos" + + def write(self): + super(VirtualDosDisk, self).write() + self.parent.set_mbr(self.dev) + +class VirtualGptDisk(VirtualDisk, GptDisk): + parttype = "gpt" + +class VirtualLvm(SizeableDisk, LvmDisk): + + class Volume(object): + def __init__(self, dev=None, mount=None, size=None): + self.dev = dev + self.mount = mount + self.size = size + self.format = "" + + def __init__(self, parent, devices, vgname, vgsize, pesize=None): + self.parent = parent + self.devices = devices + self.disk_size = vgsize + self.vgname = vgname + self.volumes = [] + super(VirtualLvm, self).__init__(vgsize) + + volume_types_map = {} + + def get_partition_name(self, name): + return "/dev/%s/%s" % (self.vgname, name) + + def create_lvm(self): + self.volumes = [] + + def create_logical_volume(self, lvname, size): + real_size = self.size(size) + self.occupe_size(real_size) + self.volumes.append( + self.Volume( + dev=self.get_partition_name(lvname), + mount="", size=real_size) + ) + + def _get_partition(self, name): + if name is None: + return self.volumes[-1] + else: + fullname = self.get_partition_name(name) + for volume in self.volumes: + if fullname == volume.dev: + return volume + raise VolumesError(_("Failed to find %s volume") % name) + + def change_purpose(self, purpose, partnum=None): + part = self._get_partition(partnum) + + part.mount = self.parent.get_mount_point(purpose) + part.format = self.parent.get_format(purpose) + + if purpose is VolumesBuilder.Purpose.UpdateRoot: + self.parent.dev_from = part.dev + + def write(self): + self.check_free_size() + devtype = self.parent.get_device_type(self.devices[0]) + for partition in self.volumes: + self.parent.disk_dev.append(partition.dev) + self.parent.disk_mount.append(partition.mount) + self.parent.disk_part.append("") + self.parent.disk_size.append(partition.size) + self.parent.disk_type.append("%s-lvm" % devtype) + self.parent.disk_format.append(partition.format) + + +class Fdisk(SizeableDisk): + __metaclass__ = ABCMeta + + def __init__(self, dev, disk_size): + super(Fdisk, self).__init__(disk_size) + self.fdisk_process = process(self.fdisk_cmd, dev) + self.fdisk = [] + self.dev = dev + self.partnumber = 0 + + volume_types_map = {} + + @abstractmethod + def create_table(self): + pass + + @property + def fdisk_cmd(self): + cmd = "/sbin/fdisk" + _fdisk_cmd = getProgPath(cmd) + if not _fdisk_cmd: + raise VolumesError(_("Command not found '%s'") % cmd) + return _fdisk_cmd + + def _fdisksize(self, size): + if size is not self.AllFree: + if size < 0: + bytesize = self.free_size + size + else: + bytesize = size + return "+%d" % (Sizes().to_Sector(bytesize)-1) + else: + return "" + + def change_partition_type(self, partid, partnum=None): + if self.partnumber < 2: + self.fdisk.append("t\n%s\n" % partid) + else: + if partnum is None: + self.fdisk.append("t\n\n%s\n" % partid) + else: + self.fdisk.append("t\n%d\n%s\n" % (partnum, partid)) + + def write(self): + self.fdisk_process.write("%sw\nq\n" % "".join(self.fdisk)) + self.fdisk_process.success() + self.update_kernel_devfs() + + def update_kernel_devfs(self): + partprobe = getProgPath("/usr/sbin/partprobe") + if partprobe: + process(partprobe, self.dev).success() + + def get_partition_name(self, num): + if self.dev[-1].isdigit(): + return ("%sp%%d" % self.dev) % num + else: + return ("%s%%d" % self.dev) % num + + +class DosFdisk(Fdisk, DosDisk): + """ + Fdisk для создания dos таблицы разделов + """ + header_size = Sizes().M + + def create_table(self): + self.fdisk = [] + self.partnumber = 0 + self.fdisk.append("o\n") + + def create_primary_partition(self, size): + end = self._fdisksize(size) + self.occupe_size(size) + self.fdisk.append("n\np\n\n\n%s\n" % end) + self.partnumber += 1 + + def create_logical_partition(self, size): + end = self._fdisksize(size) + self.occupe_size(size) + self.fdisk.append("n\n\n%s\n" % end) + self.partnumber += 1 + + def create_extended_partition(self): + self.fdisk.append("n\ne\n\n\n") + self.partnumber += 1 + + +class GptFdisk(Fdisk, GptDisk): + """ + Fdisk для создания dos таблицы разделов + """ + header_size = Sizes().M + Sizes().Sector * 34 + + def create_table(self): + self.fdisk = [] + self.fdisk.append("g\n") + + def create_partition(self, size): + end = self._fdisksize(size) + self.occupe_size(size) + self.fdisk.append("n\n\n\n%s\n" % end) + self.partnumber += 1 + + +class LvmVolumeGroup(SizeableDisk, LvmDisk): + + def __init__(self, devices, vgname, vgsize, extsize=None): + super(LvmVolumeGroup, self).__init__(vgsize) + self.devices = devices + self.vgname = vgname + self.vgsize = vgsize + self.extsize = extsize + + @property + def lvm_cmd(self): + cmd = "/sbin/lvm" + _lvm_cmd = getProgPath(cmd) + if not _lvm_cmd: + raise VolumesError(_("Command not found '%s'") % cmd) + return _lvm_cmd + + def create_physical_volumes(self): + p = process(self.lvm_cmd, "pvcreate", "-ff", *self.devices) + p.success() + + def create_volumes_group(self): + if self.extsize: + p = process(self.lvm_cmd, "vgcreate", self.vgname, + *(["-s%s" % self.extsize] + self.devices)) + else: + p = process(self.lvm_cmd, "vgcreate", self.vgname, + *self.devices) + p.success() + + def create_lvm(self): + self.create_physical_volumes() + self.create_volumes_group() + + def create_logical_volume(self, lvname, size): + size_param = self._lvmsize(size) + self.occupe_size(size) + process(self.lvm_cmd, "lvcreate", size_param, + self.vgname, "-n", lvname).success() + + def get_partition_name(self, name): + return "/dev/%s/%s" % (self.vgname, name) + + def _lvmsize(self, size): + if size is not self.AllFree: + if size < 0: + bytesize = self.free_size + size + else: + bytesize = size + return "-L%ss" % (Sizes().to_Sector(bytesize)-1) + else: + return "-l100%FREE" + + def change_purpose(self, purpose, partnum=None): + pass + + def write(self): + pass + + +class DosPartitionId(object): + volume_types_map = { + VolumesBuilder.VolumeType.Ntfs: "07", + VolumesBuilder.VolumeType.Extended: "05", + VolumesBuilder.VolumeType.Linux: "83", + VolumesBuilder.VolumeType.LinuxSwap: "82", + VolumesBuilder.VolumeType.Fat32: "0b", + VolumesBuilder.VolumeType.Lvm: "8e", + VolumesBuilder.VolumeType.Raid: "fd", + VolumesBuilder.VolumeType.Generic: "83", + } + + @classmethod + def get(cls, _id, fallback=VolumesBuilder.VolumeType.Generic): + return cls.volume_types_map.get( + _id, cls.volume_types_map[fallback]) + + +class GptPartitionId(object): + volume_types_map = { + VolumesBuilder.VolumeType.Ntfs: "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7", + VolumesBuilder.VolumeType.Fat32: "ebd0a0a2-b9e5-4433-87c0-68b6b72699c7", + VolumesBuilder.VolumeType.Linux: "0fc63daf-8483-4772-8e79-3d69d8477de4", + VolumesBuilder.VolumeType.LinuxSwap: + "0657fd6d-a4ab-43c4-84e5-0933c84b4f4f", + VolumesBuilder.VolumeType.Raid: "a19d880f-05fc-4d3b-a006-743f0f84911e", + VolumesBuilder.VolumeType.Lvm: "e6d6d379-f507-44c2-a23c-238f2a3df928", + VolumesBuilder.VolumeType.Efi: "c12a7328-f81f-11d2-ba4b-00a0c93ec93b", + VolumesBuilder.VolumeType.BiosBoot: + "21686148-6449-6e6f-744e-656564454649", + VolumesBuilder.VolumeType.Generic: + "0fc63daf-8483-4772-8e79-3d69d8477de4", + } + + @classmethod + def get(cls, _id, fallback=VolumesBuilder.VolumeType.Generic): + return cls.volume_types_map.get( + _id, cls.volume_types_map[fallback]) + + +class RoundableVolumesBuilder(VolumesBuilder): + rounding = 0.05 + free_size = 0 + last_partition = False + sizes_map = {} + minimal = 0 + + def is_roundable_size(self, size): + if (size is not VolumesBuilder.SizeAllFree and size and + abs(size - self.free_size) < size * self.rounding): + return True + return False + + def refined_size(self, size): + if size is VolumesBuilder.SizeAllFree and self.minimal: + if self.minimal > self.free_size: + return self.minimal + if self.last_partition: + if self.is_roundable_size(size): + size = VolumesBuilder.SizeAllFree + if size in self.sizes_map: + return self.sizes_map[size] + return size + + def create_last_volume(self, size, name, + purpose=VolumesBuilder.Purpose.Undefined): + self.last_partition = True + return self.create_volume(size, name, purpose) + + def set_minimal_volume_size(self, size): + self.minimal = size + + +class LvmCalculator(object): + def __init__(self, parent): + self.parent = parent + self.size = 0 + self.precreate_size = None + + def precreate(self, size, purpose): + if purpose == VolumesBuilder.Purpose.Lvm: + self.precreate_size = self.parent.free_size + else: + self.precreate_size = None + + def postcreate(self, size, purpose): + if self.precreate_size and purpose == VolumesBuilder.Purpose.Lvm: + self.size += self.precreate_size - self.parent.free_size + self.precreate_size = None + +class DosBuilder(RoundableVolumesBuilder): + def __init__(self, dosdisk): + self.dosdisk = dosdisk + self.sizes_map = {VolumesBuilder.SizeAllFree: dosdisk.AllFree} + self.next_part_number = 1 + self.lvm_data = LvmCalculator(self) + + def create(self): + self.dosdisk.create_table() + + def create_volume(self, size, name, + purpose=VolumesBuilder.Purpose.Undefined): + self.lvm_data.precreate(size, purpose) + if self.next_part_number < 4: + self.create_primary_partition(size) + else: + if self.next_part_number == 4: + self.create_extended_partition() + self.next_part_number += 1 + + self.create_logical_partition(size) + self.change_purpose(purpose) + self.next_part_number += 1 + self.lvm_data.postcreate(size, purpose) + return self.dosdisk.get_partition_name(self.next_part_number - 1) + + @property + def free_size(self): + return self.dosdisk.free_size + + def change_purpose(self, purpose): + parttype = VolumesBuilder.purpose_type.get( + purpose, + VolumesBuilder.purpose_type[VolumesBuilder.Purpose.Undefined]) + volume_id = DosPartitionId.get(parttype) + self.dosdisk.change_partition_type(volume_id) + + def create_primary_partition(self, size): + self.dosdisk.create_primary_partition(self.refined_size(size)) + + def create_logical_partition(self, size): + self.dosdisk.create_logical_partition(self.refined_size(size)) + + def create_extended_partition(self): + self.dosdisk.create_extended_partition() + + def finish(self): + self.dosdisk.write() + + +class GptBuilder(RoundableVolumesBuilder): + def __init__(self, gptdisk, table_rules): + self.gptdisk = gptdisk + self.table_rules = table_rules + self.table_rules.parent = self + self.sizes_map = {VolumesBuilder.SizeAllFree: self.gptdisk.AllFree} + self.next_part_number = 1 + self.lvm_data = LvmCalculator(self) + + def create(self): + self.gptdisk.create_table() + + def create_volume(self, size, name, + purpose=VolumesBuilder.Purpose.Undefined): + return self.table_rules.create_volume(size, purpose) + + def get_last_partition_name(self): + return self.gptdisk.get_partition_name(self.next_part_number - 1) + + @property + def free_size(self): + return self.gptdisk.free_size + + def create_partition(self, size, purpose): + self.lvm_data.precreate(size, purpose) + self.gptdisk.create_partition(self.refined_size(size)) + self.lvm_data.postcreate(size, purpose) + self.change_purpose(purpose) + self.next_part_number += 1 + + def change_purpose(self, purpose): + parttype = VolumesBuilder.purpose_type.get( + purpose, + VolumesBuilder.purpose_type[VolumesBuilder.Purpose.Undefined]) + volume_id = GptPartitionId.get(parttype) + self.gptdisk.change_partition_type(volume_id) + + def finish(self): + self.table_rules.finish() + self.gptdisk.write() + +class GptRules(object): + parent = None + + def finish(self): + pass + +class GptOnlyRules(GptRules): + def create_volume(self, size, purpose): + self.parent.create_partition(size, purpose) + return self.parent.get_last_partition_name() + +class GptFirstBootRules(GptRules): + boot_purpose = VolumesBuilder.Purpose.BiosBoot + + def __init__(self, boot_size): + self.boot_created = False + self.boot_size = boot_size + + def create_boot(self, size=None): + minimal = self.parent.minimal + self.parent.minimal = 0 + self.parent.create_partition(size or self.boot_size, + self.boot_purpose) + self.boot_created = True + self.parent.minimal = minimal + + def create_volume(self, size, purpose): + if not self.boot_created: + if purpose != self.boot_purpose: + self.create_boot() + self.boot_created = True + self.parent.create_partition(size, purpose) + return self.parent.get_last_partition_name() + +def EfiGpt(obj, size=None): + obj.boot_purpose = VolumesBuilder.Purpose.EfiBoot + if size: + obj.boot_size = size + return obj + +def BiosBoot(obj, size=None): + obj.boot_purpose = VolumesBuilder.Purpose.BiosBoot + if size: + obj.boot_size = size + return obj + +class GptPosBootRules(GptFirstBootRules): + def __init__(self, boot_size, part_pos=4): + super(GptPosBootRules, self).__init__(boot_size) + self.part_pos = part_pos + + def create_volume(self, size, purpose): + if purpose == self.boot_purpose: + self.boot_created = True + if not self.boot_created: + if self.parent.next_part_number == self.part_pos: + if purpose != self.boot_purpose: + self.create_boot() + elif self.parent.last_partition: + if (size is VolumesBuilder.SizeAllFree or + self.parent.is_roundable_size(size)): + self.parent.create_partition(-self.boot_size, purpose) + volname = self.parent.get_last_partition_name() + self.create_boot(VolumesBuilder.SizeAllFree) + else: + self.parent.create_partition(size, purpose) + volname = self.parent.get_last_partition_name() + self.create_boot(self.boot_size) + return volname + self.parent.create_partition(size, purpose) + return self.parent.get_last_partition_name() + + def finish(self): + if not self.boot_created: + self.create_boot(self.boot_size) + + +class LvmBuilder(RoundableVolumesBuilder): + """ + Сборщик разделов для LVM + """ + + def __init__(self, lvmdisk): + self.lvmdisk = lvmdisk + self.sizes_map = {VolumesBuilder.SizeAllFree: lvmdisk.AllFree} + + def create(self): + self.lvmdisk.create_lvm() + + @property + def free_size(self): + return self.lvmdisk.free_size + + def create_volume(self, size, name, + purpose=VolumesBuilder.Purpose.Undefined): + self.lvmdisk.create_logical_volume(name, self.refined_size(size)) + self.change_purpose(purpose) + return self.lvmdisk.get_partition_name(name) + + def change_purpose(self, purpose): + pass + + def finish(self): + self.lvmdisk.write() + + +class VolumesSet(object): + def __init__(self, builder): + self.builder = builder + self.params = [] + + def create_volume(self, size, name, purpose, minimal_size=None): + self.params.append((size, name, purpose, minimal_size)) + + def __enter__(self): + self.builder.create() + return self + + def execute(self): + for mark, data in classificate(self.params): + size, name, purpose, minsize = data + if minsize: + self.builder.set_minimal_volume_size(minsize) + if mark.last: + self.builder.create_last_volume(size, name, purpose) + else: + self.builder.create_volume(size, name, purpose) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.execute() + self.builder.finish() + + +class BuilderFactory(object): + __metaclass__ = ABCMeta + + @abstractmethod + def createDosBuilder(self, device, size): + pass + + @abstractmethod + def createGptBuilder(self, device, size, table_rules): + pass + + @abstractmethod + def createLvmBuilder(self, devices, vgname, vgsize, extsize=None): + pass + +class DiskFactory(BuilderFactory): + def createDosBuilder(self, device, size): + return DosBuilder(DosFdisk(device, size)) + + def createGptBuilder(self, device, size, table_rules): + return GptBuilder(GptFdisk(device, size), + table_rules) + + def createLvmBuilder(self, devices, vgname, vgsize, extsize=None): + return LvmBuilder(LvmVolumeGroup(devices, vgname, vgsize, + extsize=extsize)) + + +class DeviceInfo(object): + __metaclass__ = ABCMeta + + @abstractmethod + def get_device_type(self, path=None, name=None): + pass + +class PurposedDosBuilder(DosBuilder): + def change_purpose(self, purpose): + super(PurposedDosBuilder, self).change_purpose(purpose) + self.dosdisk.change_purpose(purpose) + +class PurposedGptBuilder(GptBuilder): + def change_purpose(self, purpose): + super(PurposedGptBuilder, self).change_purpose(purpose) + self.gptdisk.change_purpose(purpose) + +class PurposedLvmBuilder(LvmBuilder): + def change_purpose(self, purpose): + super(PurposedLvmBuilder, self).change_purpose(purpose) + self.lvmdisk.change_purpose(purpose) + +class VariableFactory(BuilderFactory): + def __init__(self, deviceinfo): + self.disk_dev = [] + self.disk_mount = [] + self.disk_part = [] + self.disk_size = [] + self.disk_type = [] + self.dev_from = "" + self.disk_format = [] + self.deviceinfo = deviceinfo + self.lvm_size = {} + self.lvm_disk_type = {} + + self.mbr = [] + self.efi = [] + + self.purpose_mount = { + VolumesBuilder.Purpose.Calculate: "/var/calculate", + VolumesBuilder.Purpose.MainRoot: "/", + VolumesBuilder.Purpose.Swap: "swap", + } + self.purpose_format = { + VolumesBuilder.Purpose.Swap: "swap", + VolumesBuilder.Purpose.EfiBoot: "vfat", + VolumesBuilder.Purpose.Windows: "ntfs", + } + self.default_format = "ext4" + + self.pesize = None + + def get_mount_point(self, purpose): + if not self.efi and purpose == VolumesBuilder.Purpose.EfiBoot: + return "/boot/efi" + return self.purpose_mount.get(purpose, '') + + def get_format(self, purpose): + return self.purpose_format.get(purpose, self.default_format) + + def createDosBuilder(self, device, size): + return PurposedDosBuilder(VirtualDosDisk(self, device, size)) + + def createGptBuilder(self, device, size, table_rules): + return PurposedGptBuilder(VirtualGptDisk(self, device, size), + table_rules) + + def createLvmBuilder(self, devices, vgname, vgsize, extsize=None): + return PurposedLvmBuilder(VirtualLvm(self, devices, vgname, vgsize, + pesize=self.pesize)) + + def get_device_type(self, name): + if name in self.lvm_disk_type: + return self.lvm_disk_type[name] + return self.deviceinfo.get_device_type(name=name) + + def set_mbr(self, device): + self.mbr.append(device) + + def set_efi(self, device): + self.efi.append(device) + + +class SchemeError(Exception): + pass + +class DeviceSchemeError(SchemeError): + pass + +class SchemeBuilder(object): + class PartitionTable(object): + GPT = 0 + DOS = 1 + + def __init__(self): + self.efi = False + self.partition_table = self.PartitionTable.DOS + self.lvm = False + self.devices = [] + self.swap_size = Sizes().G + self.root_size = 10 * Sizes().G + self.efi_size = 200 * Sizes().M + self.biosboot_size = 50 * Sizes().M + self.swap = False + self.minimal_lvm_size = 1 * Sizes().G + self.minimal_calculate_size = 500 * Sizes().M + self.update = False + self.calculate = False + self.rounding = 0.01 + self.vgname = "calculate" + + self.extsize = None + + def add_device(self, device, _type, size): + self.devices.append((device, _type, size)) + + def create_lvm_devices(self, volumes_factory): + for dev, _type, size in self.devices: + part_rules = GptFirstBootRules(self.biosboot_size) + if self.efi: + part_rules = EfiGpt(part_rules, size=self.efi_size) + else: + part_rules = BiosBoot(part_rules, size=self.biosboot_size) + if _type == VolumesBuilder.DeviceType.RAID: + yield dev, size + else: + if self.partition_table is SchemeBuilder.PartitionTable.DOS: + builder = volumes_factory.createDosBuilder(dev, size) + else: + builder = volumes_factory.createGptBuilder( + dev, size, part_rules) + builder.create() + builder.set_minimal_volume_size(self.minimal_lvm_size) + devname = builder.create_last_volume( + VolumesBuilder.SizeAllFree, "lvm", + VolumesBuilder.Purpose.Lvm) + builder.finish() + devsize = builder.lvm_data.size + yield devname, devsize + + def get_volume_builder(self, volumes_factory): + if not self.devices: + raise DeviceSchemeError( + _("Please select devices for partitions changing")) + if self.lvm: + lvm_devices, lvm_sizes = zip( + *self.create_lvm_devices(volumes_factory)) + return volumes_factory.createLvmBuilder( + list(lvm_devices), self.vgname, + sum(lvm_sizes), extsize=self.extsize) + else: + if len(self.devices) > 1: + raise DeviceSchemeError( + _("You should use LVM to install on more that one device")) + dev, devtype, size = self.devices[0] + if self.partition_table is SchemeBuilder.PartitionTable.DOS: + return volumes_factory.createDosBuilder(dev, size) + else: + if devtype is VolumesBuilder.DeviceType.RAID: + return volumes_factory.createGptBuilder( + dev, size, GptOnlyRules()) + else: + part_rules = GptPosBootRules("", 4) + if self.efi: + part_rules = EfiGpt(part_rules, size=self.efi_size) + else: + part_rules = BiosBoot(part_rules, + size=self.biosboot_size) + return volumes_factory.createGptBuilder(dev, size, + part_rules) + + def process(self, volumes_factory): + volume_builder = self.get_volume_builder(volumes_factory) + volume_builder.rounding = self.rounding + with VolumesSet(volume_builder) as builder: + if self.swap: + builder.create_volume(self.swap_size, "swap", + VolumesBuilder.Purpose.Swap) + builder.create_volume(self.root_size, "system1", + VolumesBuilder.Purpose.MainRoot, None) + if self.update: + builder.create_volume(self.root_size, "system2", + VolumesBuilder.Purpose.UpdateRoot) + if self.calculate: + builder.create_volume(VolumesBuilder.SizeAllFree, "calculate", + VolumesBuilder.Purpose.Calculate, + self.minimal_calculate_size) diff --git a/pym/calculate/lib/utils/tools.py b/pym/calculate/lib/utils/tools.py index 81c64d2..e4621ed 100644 --- a/pym/calculate/lib/utils/tools.py +++ b/pym/calculate/lib/utils/tools.py @@ -26,6 +26,7 @@ import fcntl from os import path import time from abc import ABCMeta, abstractmethod +from types import GeneratorType @@ -102,6 +103,7 @@ class Sizes(object): Gb = gigabyte = Mb * 1000 T = TiB = tibibyte = G * 1024 Tb = terabyte = Gb * 1000 + Sector = 512 def __getattr__(self, name): if name.startswith('from_'): @@ -111,6 +113,11 @@ class Sizes(object): else: raise AttributeError + def _from(self, name, count): + return count * getattr(Sizes, name) + + def _to(self, name, count): + return count / getattr(Sizes, name) def imap_regexp(re_compiled, l, whole=False): """ @@ -377,7 +384,7 @@ def max_default(iterable, key=lambda x: x, default=None): return default -def traverse(o, tree_types=(list, tuple)): +def traverse(o, tree_types=(list, tuple, GeneratorType)): """ Вернуть последовательно все элемены списка, "распаковов" встроенные