Переписана авторазметка.

Модуль install теперь используется utils/partition

pym/calculate/lib/datavars.py

@@ -1099,6 +1099,7 @@ class SimpleDataVars(object):
                 'ne': lambda x, y: lambda z: z[x] != y,
                 'in': lambda x, y: lambda z: z[x] in y,
                 'not_in': lambda x, y: lambda z: z[x] not in y,
+                'startswith': lambda x, y: lambda z: z[x].startswith(y)
             }
             filter_chain.append(func_maps[funcname](len(variables) - 1, kw[k]))
         l = len(fields)

pym/calculate/lib/utils/debug.py

@@ -15,7 +15,8 @@
 #  limitations under the License.
 
 from calculate.lib.utils.device import (Sysfs, UdevAdm, sysfs, Devfs, Udev, Lvm,
-                                        UdevAdmNull)
+                                        UdevAdmNull, LvmCommand, MdadmCommand,
+                                        Raid)
 from calculate.lib.utils.files import writeFile
 from calculate.lib.utils.vfs import (VFSKeeper, VFSSysfsImporter,
                                      SafeVFS,
@@ -71,25 +72,55 @@ class VirtualUdevAdm(UdevAdmNull):
         pass
 
 
-class VirtualLvm(Lvm):
+class VirtualLvmCommand(object):
     def __init__(self, output):
-        super(VirtualLvm, self).__init__()
         self.output = output
 
-    def pvdisplay_output(self, output):
+    def get_physical_extent_size(self):
-        if output == "vg_name,lv_name,pv_name":
+        return "4096"
-            return self.output
 
-    def refresh(self):
+    def get_pvdisplay_output(self, output):
+        return self.output
+
+    def vgscan(self):
+        return True
+
+    def vgchange(self):
+        return True
+
+    def lvchange(self, groups):
+        return True
+
+    def execute(self, *command):
+        return True
+
+    def double_execute(self, *command):
+        return True
+
+    def remove_lv(self, vg, lv):
+        return True
+
+    def remove_vg(self, vg):
+        return True
+
+    def remove_pv(self, pv):
+        return True
+
+
+class VirtualMdadmCommand(object):
+    def zero_superblock(self, devices):
         pass
 
+    def stop_raid(self, devraid):
+        pass
 
 class VirtualDeviceModule(object):
     def __init__(self):
         self.sysfs = Sysfs()
         self.devfs = Devfs()
         self.udev = Udev()
-        self.lvm = Lvm()
+        self.lvm = Lvm(LvmCommand())
+        self.raid = Raid(MdadmCommand())
 
     def set_sysfs(self, f):
         self.sysfs = Sysfs(SafeVFS(VFSKeeper.load(f)))
@@ -100,11 +131,14 @@ class VirtualDeviceModule(object):
     def set_udev(self, f):
         self.udev.udevadm = VirtualUdevAdm(f.read(), self)
 
+    def set_raid(self):
+        self.raid = Raid(VirtualMdadmCommand())
+
     def set_lvm(self, f):
         if f is None:
-            self.lvm = VirtualLvm("")
+            self.lvm = Lvm(VirtualLvmCommand(""))
         else:
-            self.lvm = VirtualLvm(f.read())
+            self.lvm = Lvm(VirtualLvmCommand(f.read()))
 
 class DumpError(Exception):
     pass

pym/calculate/lib/utils/device.py

@@ -18,8 +18,9 @@ import sys
 import re
 import os
 from os import path
-from calculate.lib.utils.tools import (Cachable, GenericFs, sorteduniqresult)
+from calculate.lib.utils.tools import (Cachable, GenericFs, unique)
 import files
+from time import sleep
 
 from calculate.lib.cl_lang import setLocalTranslate
 
@@ -60,27 +61,89 @@ def countPartitions(devname):
     return len([x for x in sysfs.listdir(syspath) if x.startswith(device_name)])
 
 
+class LvmCommand(object):
+    @property
+    def lvm_cmd(self):
+        return files.getProgPath('/sbin/lvm')
+
+    def get_physical_extent_size(self):
+        if not self.lvm_cmd:
+            return ""
+        pvdata = files.process(self.lvm_cmd, "lvmconfig", "--type", "full",
+                               "allocation/physical_extent_size")
+        if pvdata.success():
+            return pvdata.read().strip().rpartition("=")[-1]
+        return ""
+
+    def get_pvdisplay_output(self, output):
+        if not self.lvm_cmd:
+            return ""
+        pvdata = files.process(self.lvm_cmd, "pvdisplay", "-C", "-o",
+                               output, "--noh")
+        if pvdata.success():
+            return pvdata.read()
+        return ""
+
+    def vgscan(self):
+        if self.lvm_cmd:
+            return files.process(self.lvm_cmd, "vgscan").success()
+        return False
+
+    def vgchange(self):
+        failed = True
+        if self.lvm_cmd:
+            failed = files.process("vgchange", '-ay').failed()
+            failed |= files.process("vgchange", '--refresh').failed()
+        return not failed
+
+    def lvchange(self, groups):
+        failed = True
+        if self.lvm_cmd:
+            failed = False
+            for group in groups:
+                failed |= files.process("lvchange", '-ay', group).failed()
+                failed |= files.process("lvchange", '--refresh', group).failed()
+        return not failed
+
+    def execute(self, *command):
+        if self.lvm_cmd:
+            return files.process(self.lvm_cmd, *command).success()
+        return False
+
+    def double_execute(self, *command):
+        if not self.execute(*command):
+            sleep(2)
+            return self.execute(*command)
+        return False
+
+    def remove_lv(self, vg, lv):
+        return self.double_execute("lvremove", "%s/%s" % (vg, lv), "-f")
+
+    def remove_vg(self, vg):
+        return self.double_execute("vgremove", vg, "-f")
+
+    def remove_pv(self, pv):
+        return self.double_execute("pvremove", pv, "-ffy")
+
 class Lvm(Cachable):
     """
     LVM информация
     """
-    @property
+    def __init__(self, commander):
-    def pvdisplay_cmd(self):
+        super(Lvm, self).__init__()
-        return files.getProgPath('/sbin/pvdisplay')
+        self.commander = commander
 
     @property
     def groups(self):
         return sorted({vg for vg, lv, pv in self.pvdisplay_full()})
 
+    @Cachable.methodcached()
+    def get_pesize(self):
+        return self.commander.get_physical_extent_size()
+
     @Cachable.methodcached()
     def pvdisplay_output(self, output):
-        if not self.pvdisplay_cmd:
+        return self.commander.get_pvdisplay_output(output)
-            return ""
-        pvdata = files.process(self.pvdisplay_cmd, "-C", "-o",
-                               output, "--noh")
-        if pvdata.success():
-            return pvdata.read()
-        return ""
 
     def pvdisplay_full(self):
         data = self.pvdisplay_output("vg_name,lv_name,pv_name").strip()
@@ -96,18 +159,19 @@ class Lvm(Cachable):
 
     def refresh(self):
         """Run command which refresh information about LVM"""
-        vgscan = files.getProgPath('/sbin/vgscan')
+        if not os.environ.get('EBUILD_PHASE'):
-        vgchange = files.getProgPath('/sbin/vgchange')
+            self.commander.vgscan()
-        lvchange = files.getProgPath('/sbin/lvchange')
+            self.commander.vgchange()
+            self.commander.lvchange(lvm.groups)
 
-        if vgscan and vgchange and lvchange:
+    def remove_lv(self, vg, lv):
-            files.process(vgscan).success()
+        return self.commander.remove_lv(vg, lv)
-            files.process(vgchange, '-ay').success()
+
-            files.process(vgchange, '--refresh').success()
+    def remove_vg(self, vg):
-            for group in lvm.groups:
+        return self.commander.remove_vg(vg)
-                if not os.environ.get('EBUILD_PHASE'):
+
-                    files.process(lvchange, '-ay', group).success()
+    def remove_pv(self, pv):
-                    files.process(lvchange, '--refresh', group).success()
+        return self.commander.remove_pv(pv)
 
 
 def lspci(filtername=None, shortInfo=False):
@@ -259,14 +323,22 @@ class Udev(object):
         """
         Получить все устройства SUBSYSTEM=block устройства
         """
-        block_entries = (x for x in self.udevadm.info_export().split("\n\n")
+        for block in sysfs.glob(sysfs.Path.Block, "*"):
-                         if "SUBSYSTEM=block" in x)
+            yield block
-        for block in block_entries:
+            blockname = path.basename(block)
-            for entry in (x[3:] for x in block.split("\n")
+            for part in sysfs.glob(block, "%s*" % blockname):
-                          if x.startswith("P:")):
+                yield part
-                if "/block/" in entry:
+
-                    yield entry[entry.rindex("/block/"):]
+        #block_devices = {sysfs.realpath(x)[len(sysfs.base_dn):]: x
-                break
+        #                 for x in sysfs.listdir("/block/*", fullpath=True)}
+        #block_entries = (x for x in self.udevadm.info_export().split("\n\n")
+        #                 if "SUBSYSTEM=block" in x)
+        #for block in block_entries:
+        #    for entry in (x[3:] for x in block.split("\n")
+        #                  if x.startswith("P:")):
+        #        if "/block/" in entry:
+        #            yield entry[entry.rindex("/block/"):]
+        #        break
 
     def syspath_to_devname(self, it, dropempty=True):
         """
@@ -371,33 +443,63 @@ class Udev(object):
                     return "primary"
         return info.get('ID_PART_TABLE_TYPE', '')
 
-    @sorteduniqresult
+    def _get_disk_devices(self, path=None, name=None):
-    def get_disk_devices(self, path=None, name=None):
+        """
-        """Get real parent device by partition,lvm,mdraid"""
+        Возвращает список базовых устройств (признак базовое ли устройство,
+        название устройства)
+        :param path:
+        :param name:
+        :return:
+        """
         info = udev.get_device_info(path=path, name=name)
         syspath = info.get("DEVPATH", '')
         if syspath:
             # real device
             if self.is_device(info):
-                yield info.get("DEVNAME", "")
+                yield True, info.get("DEVNAME", "")
             # partition
             elif self.is_partition(info):
-                for x in self.get_disk_devices(path=os.path.dirname(syspath)):
+                for x in self._get_disk_devices(path=os.path.dirname(syspath)):
                     yield x
             # md raid
             elif udev.is_raid(info):
-                for rd in raid.devices_syspath(syspath):
+                yield False, info.get("DEVNAME", "")
-                    for x in self.get_disk_devices(path=rd):
+                for rd in sorted(raid.devices_syspath(syspath)):
+                    for x in self._get_disk_devices(path=rd):
                         yield x
             # lvm
             elif udev.is_lvm(info):
+                yield False, info.get("DEVNAME", "")
                 for lvdev in lvm.used_partitions(info["DM_VG_NAME"],
                                                  info["DM_LV_NAME"]):
-                    for x in self.get_disk_devices(name=lvdev):
+                    for x in self._get_disk_devices(name=lvdev):
                         yield x
 
+    def get_disk_devices(self, path=None, name=None):
+        """Get real parent device by partition,lvm,mdraid"""
+        return sorted({
+            dev
+            for realdevice, dev in self._get_disk_devices(path, name)
+            if realdevice
+            })
 
-def humanreadableSize(size):
+    def get_all_base_devices(self, path=None, name=None):
+        """
+        Получить все устройства (включая RAID и LVM) из которого состоит
+        указанное устройство, исключая loop устройства
+        """
+        try:
+            devs = (dev
+                    for realdevice, dev in self._get_disk_devices(path, name)
+                    if not dev.startswith("/dev/loop"))
+            if not self.is_partition(self.get_device_info(path, name)):
+                next(devs)
+            return list(unique(devs))
+        except StopIteration:
+            return []
+
+
+def humanreadableSize(size, compsize=0):
     """
     Human readable size (1024 -> 1K and etc)
     """
@@ -411,7 +513,7 @@ def humanreadableSize(size):
               ((1024 ** 3), "G", True),
               ((1024 ** 4), "T", True),
               ((1024 ** 5), "P", True))
-    suffix = filter(lambda x: size > x[0], suffix)
+    suffix = filter(lambda x: abs(size-compsize) > x[0], suffix)
     if suffix:
         suffix = suffix[-1]
         printSize = int(size / (float(suffix[0]) / 10))
@@ -442,14 +544,32 @@ def getPartitionSize(syspath=None, name=None, inBytes=False):
     return ""
 
 
+class MdadmCommand(object):
+    @property
+    def mdadm_cmd(self):
+        return files.getProgPath('/sbin/mdadm')
+
+    def stop_raid(self, devraid):
+        if not self.mdadm_cmd:
+            return ""
+        pvdata = files.process(self.mdadm_cmd, "-S", devraid)
+        return pvdata.success()
+
+    def zero_superblock(self, devices):
+        if not self.mdadm_cmd:
+            return ""
+        for dev in devices:
+            files.process(self.mdadm_cmd, "--zero-superblock", dev).success()
+
 class Raid(object):
+    def __init__(self, commander):
+        self.commander = commander
+
     def devices_info(self, raid):
         prop = udev.get_device_info(path=raid)
         if udev.is_raid(prop):
-            for rd in (x for x in sysfs.listdir(raid, "md", fullpath=True)
+            for rdblock in sysfs.glob(raid, "md/dev-*", "block"):
-                       if path.basename(x).startswith('rd')):
+                yield udev.get_device_info(sysfs.syspath(rdblock))
-                if sysfs.exists(rd, "block"):
-                    yield udev.get_device_info(sysfs.syspath(rd, "block"))
 
     def devices_syspath(self, raid):
         """
@@ -462,7 +582,8 @@ class Raid(object):
 
     def devices(self, raid, pathname="DEVNAME"):
         """
-        Получить устройства /dev
+        Получить устройства /dev из которых состоит RAID, не возвращает
+        список этих устройств для раздела сделанного для RAID (/dev/md0p1)
         :param raid:
         :param pathname:
         :return:
@@ -472,6 +593,13 @@ class Raid(object):
             if devname:
                 yield devname
 
+    def remove_raid(self, raidname):
+        raidparts = self.devices(udev.get_syspath(name=raidname))
+        failed = False
+        self.commander.stop_raid(raidname)
+        for dev in raidparts:
+            self.commander.zero_superblock(dev)
+        return failed
 
 def loadEfiVars():
     """
@@ -576,5 +704,5 @@ class Devfs(DeviceFs):
 sysfs = Sysfs()
 devfs = Devfs()
 udev = Udev()
-lvm = Lvm()
+lvm = Lvm(LvmCommand())
-raid = Raid()
+raid = Raid(MdadmCommand())

pym/calculate/lib/utils/tools.py

@@ -26,6 +26,7 @@ import fcntl
 from os import path
 import time
 from abc import ABCMeta, abstractmethod
+from types import GeneratorType
 
 
 
@@ -102,6 +103,7 @@ class Sizes(object):
     Gb = gigabyte = Mb * 1000
     T = TiB = tibibyte = G * 1024
     Tb = terabyte = Gb * 1000
+    Sector = 512
 
     def __getattr__(self, name):
         if name.startswith('from_'):
@@ -111,6 +113,11 @@ class Sizes(object):
         else:
             raise AttributeError
 
+    def _from(self, name, count):
+        return count * getattr(Sizes, name)
+
+    def _to(self, name, count):
+        return count / getattr(Sizes, name)
 
 def imap_regexp(re_compiled, l, whole=False):
     """
@@ -377,7 +384,7 @@ def max_default(iterable, key=lambda x: x, default=None):
         return default
 
 
-def traverse(o, tree_types=(list, tuple)):
+def traverse(o, tree_types=(list, tuple, GeneratorType)):
     """
     Вернуть последовательно все элемены списка, "распаковов" встроенные