calculate-utils-3-lib/pym/calculate/lib/utils/files.py

#-*- coding: utf-8 -*-

# Copyright 2008-2013 Calculate Ltd. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
from os import path
import subprocess
from subprocess import Popen,PIPE,STDOUT
import stat
from shutil import copytree, rmtree,copy2
from calculate.lib import cl_overriding
import re
import sys
from itertools import *
import tarfile
import select
import fcntl

from contextlib import contextmanager
import signal

@contextmanager
def timeout(seconds):
    def timeout_handler(signum,frame):
        pass

    original_handler = signal.signal(signal.SIGALRM, timeout_handler)

    try:
        signal.alarm(seconds)
        yield
    finally:
        signal.alarm(0)
        signal.signal(signal.SIGALRM, original_handler)

try:
    from magic import (open as type_file, MAGIC_NONE, MAGIC_CONTINUE,
                        MAGIC_MIME_TYPE, MAGIC_COMPRESS,
                        MAGIC_MIME_ENCODING, MAGIC_SYMLINK)
except ImportError as e:
    try:
        from magic import (open as type_file, NONE as MAGIC_NONE,
                CONTINUE as MAGIC_CONTNUE, MIME_TYPE as MAGIC_MIME_TYPE,
                COMPRESS as MAGIC_COMPRESS,MIME_ENCODING as MIME_ENCODING,
                SYMLINK as MAGIC_SYMLINK)
    except:
        type_file = None
        MAGIC_NONE = 0
        MAGIC_SYMLINK = 0x002
        MAGIC_COMPRESS = 0x004
        MAGIC_CONTINUE = 0x020
        MAGIC_MIME_TYPE = 0x010
        MAGIC_MIME_ENCODING = 0x400

from calculate.lib.cl_lang import setLocalTranslate
setLocalTranslate('cl_lib3',sys.modules[__name__])

reSearch = lambda pattern,listar:map(lambda x:x.groups(),
                        filter(None,
                        map(pattern.search,listar)))

class FilesError(Exception):
    pass

class process:
    """Execute system command by Popen
    
    Examples:
    
    execute program and get result:
    if process("/bin/gzip","/boot/somefile").success():
        print "Gzip success"

    unzip and process unzip data by cpio (list files):
    processGzip = process("/bin/gzip","-dc","/boot/initrd")
    processCpio = process("/bin/cpio","-tf",stdin=processGzip)
    filelist = processCpio.readlines()

    execute command and send data:
    processGrub = process("/sbin/grub")
    processGrub.write("root (hd0,0)\n")
    processGrub.write("setup (hd0)\n")
    processGrub.write("quit\n")
    isok = processGrub.success()

    union stdout and stderr:
    process("/bin/ls","/",stderr=STDOUT)
    
    result to stdout:
    process("/bin/ls",stdout=None)

    get data from keyboard:
    process("/bin/cat",stdin=None)
    """
    def __init__(self,command,*params,**kwarg):
        if not "stdin" in kwarg:
            stdin=self._defaultStdin
        else:
            if kwarg["stdin"] == None:
                stdin = self._keyboardStdin
            else:
                stdin=kwarg["stdin"].getStdout
        self.stdout = kwarg.get("stdout",PIPE)
        self.envdict = kwarg.get("envdict",os.environ.copy())
        self.envdict["LANG"] = kwarg.get('lang','C')
        self.langc = "langc" in kwarg

        self.stderr = kwarg.get("stderr",PIPE)
        self.cwd = kwarg.get("cwd",None)
        self.command = getProgPath(command)
        if not self.command:
            raise FilesError(_("Command not found '%s'")%
                             command)
        self.command = [self.command] + list(params)
        self.stdin = stdin
        self.iter = None
        self.pipe = None
        self.cacheresult = None
        self.cacheerr = None
        self.communicated = False

    def _open(self):
        """Open pipe if it not open"""
        if not self.pipe:
            self.pipe = Popen(self.command,
                              stdout=self.stdout,
                              stdin=self.stdin(),
                              stderr=self.stderr,
                              cwd=self.cwd,
                              close_fds=True,
                              env=self.envdict)

    def _defaultStdin(self):
        """Return default stdin"""
        return PIPE

    def _keyboardStdin(self):
        """Return keyboard stdin"""
        return None

    def getStdout(self):
        """Get current stdout"""
        self.close()
        self._open()
        return self.pipe.stdout

    def write(self,data):
        """Write to process stdin"""
        self._open()
        self.pipe.stdin.write(data)
        self.pipe.stdin.flush()

    def close(self):
        """Close stdin"""
        if self.pipe:
            self.pipe.stdin.close()

    def readerr(self):
        self.read()
        return self.cacheerr

    def readByLine(self):
        
        try:
            self._open()
            if self.cacheresult is None:
                self.cacheresult = []
                s = self.pipe.stdout.readline()
                while s:
                    yield s
                    s = self.pipe.stdout.readline()
                if self.pipe.stderr:
                    self.cacheerr = self.pipe.stderr.read()
                else:
                    self.cacheerr = ""

        except KeyboardInterrupt:
            self.kill()
            raise KeyboardInterrupt
        finally:
            self.cacheresult = "\n".join(self.cacheresult)

    def read(self):
        """Read all data"""
        try:
            self._open()
            if self.cacheresult is None:
                self.cacheresult,self.cacheerr = self.pipe.communicate()
            return self.cacheresult
        except KeyboardInterrupt:
            self.kill()
            raise KeyboardInterrupt

    def readlines(self):
        """Read lines"""
        return self.read().split('\n')

    def __iter__(self):
        """Get iterator"""
        if not self.iter:
            self.iter = iter(self.readlines())
        return self.iter

    def kill(self):
        """Kill this process"""
        if self.pipe:
            self.pipe.kill()

    def next(self):
        """Next string from stdout"""
        return self.__iter__().next()

    def returncode(self):
        """Get return code"""
        self.read()
        return self.pipe.returncode

    def success(self):
        """Success or not"""
        return self.returncode() == 0

    def failed(self):
        """Failed or not"""
        return self.returncode() != 0


def getModeFile(nameFile, mode="all"):
    """Выдает информацию о файле
    mode=="all"
        права файла, владелец, группа файла
    mode=="mode"
        права файла
    mode=="owner"
        владелец, группа файла
    """
    fst = os.lstat(nameFile)
    if mode == "all":
        return (stat.S_IMODE(fst.st_mode), fst.st_uid, fst.st_gid)
    if mode == "mode":
        return stat.S_IMODE(fst.st_mode)
    if mode == "owner":
        return fst.st_uid, fst.st_gid

def chownR(directory, uid, gid):
    """Recusive chown"""
    def chownPaths(rootPath, listPath, uid, gid):
        for chPath in listPath:
            chownPath = path.join(rootPath, chPath)
            statInfo = os.lstat(chownPath)[stat.ST_MODE]
            if stat.S_ISLNK(statInfo):
                os.lchown(chownPath, uid, gid)
            else:
                os.chown(chownPath, uid, gid)

    for root, dirs, files in os.walk(directory):
        # меняем владельца директории
        os.chown(root, uid, gid)
        # Меняем владельца директорий
        chownPaths(root, dirs, uid, gid)
        # Меняем владельца файлов
        chownPaths(root, files, uid, gid)
    return True

class proxy_type_file:
    def __init__(self,flags):
        self.flags = flags

    def load(self):
        pass

    def setflags(self,flags):
        self.flags = flags

    def close(self):
        pass

    def _get_cmd_by_flags(self,flags):
        """
        Получить команду для выполнения по флагам
        """
        paramdict = {MAGIC_CONTINUE:"-k",
                     MAGIC_SYMLINK:"-L",
                     MAGIC_MIME_TYPE:"--mime-type",
                     MAGIC_MIME_ENCODING:"--mime-encoding",
                     MAGIC_COMPRESS:"-z"}
        appendParam = map(lambda x:paramdict[x],
                      filter(lambda x:flags & x,
                      sorted(paramdict.keys())))
        fileCmd = getProgPath('/usr/bin/file')
        return [fileCmd,'-b']+appendParam

    def file(self,filename):
        if path.exists(filename):
            processFile = \
                process(*(self._get_cmd_by_flags(self.flags)+[filename]))
            if processFile.success():
                return processFile.read().rstrip()
        return None

class typeFile:
    """Получение типа файла"""

    def __init__(self, magic=MAGIC_MIME_ENCODING|MAGIC_MIME_TYPE):
        if type_file is None:
            self.magicObject = proxy_type_file(MAGIC_NONE)
        else:
            self.magicObject = type_file(MAGIC_NONE)
        self.magicObject.load()
        self.magicObject.setflags(magic)

    def __del__(self):
        """Закрываем magic"""
        self.magicObject.close()

    def getMType(self, filename):
        """Информация о типе файла"""
        ret = self.magicObject.file(filename)
        # fix for kernel 3.7.7 (bad work samba)
        if ret is None and self.magicObject.errno() == 5:
            r,w = os.pipe()
            devnull = os.open(os.devnull,os.O_WRONLY)
            cat = subprocess.Popen(['/bin/cat',filename],stdout=w,
                    stderr=devnull,close_fds=True)
            ret = self.magicObject.descriptor(r)
            os.close(w)
            os.close(devnull)
            cat.wait()
        return ret

    def isBinary(self, filename):
        """является ли файл бинарным"""
        mime = self.getMType(filename)
        if mime.startswith("text"):
            return False
        else:
            return True

class scanDirectory:
    """Класс для cканирования директории"""

    def processingFile(self, pathname, prefix):
        """Обработка в случае файла"""
        return True

    def processingDirectory(self, pathname, prefix):
        """Обработка в случае директории если возвращаем None то пропуск дир."""
        return True

    def scanningDirectory(self, scanDir, skipFile=[], skipDir=[],
                          prefix=None, flagDir=False):
        """Сканирование и обработка шаблонов в директории scanDir"""
        ret = True
        if not prefix:
            prefix = path.join(scanDir,"")[:-1]
        if not flagDir:
            # проверка корневой директории
            retDir = self.processingDirectory(scanDir, scanDir)
            if retDir is None:
                return None
            elif retDir is False:
                return False
        if flagDir or stat.S_ISDIR(os.lstat(scanDir)[stat.ST_MODE]):
            for absPath in sorted(listDirectory(scanDir,fullPath=True)):
                relPath = absPath.split(prefix)[1]
                stInfo = os.lstat(absPath)
                statInfo = stInfo[stat.ST_MODE]
                if stat.S_ISREG(statInfo):
                    # Обработка файла
                    if relPath in skipFile:
                        continue
                    if not self.processingFile(absPath, prefix):
                        return False
                elif stat.S_ISDIR(statInfo):
                    # Обработка директории
                    if relPath in skipDir:
                        continue
                    retDir = self.processingDirectory(absPath, prefix)
                    if retDir is None:
                        continue
                    elif retDir is False:
                        return False
                    ret = self.scanningDirectory(absPath, skipFile,
                                                 skipDir, prefix, True)
                    if ret is False:
                        return False
        return ret

class processProgress(process):
    """
    Generator like process progress
    """
    def __init__(self,command,*params,**kwarg):
        process.__init__(self,command,*params,**kwarg)
        self.readsize = kwarg.get("readsize",10)
        self.delimeter = re.compile("\n")
        self.init(**kwarg)

    def init(self,*args,**kwarg):
        pass

    def processInit(self):
        pass

    def processEnd(self):
        pass

    def processString(self,strdata):
        self.cacheresult.append(strdata)
        return strdata

    def processBuffer(self,buf):
        pass

    def progress(self):
        try:
            res = self.processInit()
            if res != None: yield res
            self._open()
            if self.cacheresult is None:
                self.cacheresult = []
                self.buf = ""
                fd = self.pipe.stdout.fileno()
                fl = fcntl.fcntl(fd,fcntl.F_GETFL)
                fcntl.fcntl(fd,fcntl.F_SETFL,fl | os.O_NONBLOCK)
                hasData = True
                while hasData or self.pipe.poll() is None:
                    hasData = False
                    try:
                        part = self.pipe.stdout.read(self.readsize)
                        if not part:
                            continue
                    except IOError as e:
                        # re-raise not "Resource temporary unavailable"
                        if e.errno != 11:
                            raise
                        continue
                    hasData = True
                    if self.buf:
                        self.buf += part
                    else:
                        self.buf = part
                    res = self.processBuffer(part)
                    if res:
                        yield res
                    while self.delimeter.search(self.buf):
                        strdata, self.buf = self.delimeter.split(self.buf,1)
                        res = self.processString(strdata)
                        if res:
                            yield res
            res = self.processEnd()
            if res != None: yield res
        except KeyboardInterrupt:
            self.pipe.kill()
            raise
        finally:
            if self.cacheresult:
                self.cacheresult = "\n".join(self.cacheresult)
            else:
                self.cacheresult = ""

def countFiles(dirpath,onefilesystem=True):
    """
    Count files in specified dirpath
    """
    num = 1
    for dirpath,dirnames,filenames in os.walk(dirpath):
        num += len(set(dirnames) | set(filenames))
        if onefilesystem:
            mountDirs = filter(lambda x:path.ismount(path.join(dirpath,x)),
                        dirnames)
            for dirname in mountDirs:
                dirnames.remove(dirname)
    return num

def getFilesCount(directory):
    """Alias for compatibility"""
    return countFiles(directory,onefilesystem=False)


def runOsCommand(cmd,in_str=None, env_dict=None):
    """Run system command

    Параметры:
    cmd            внешняя программа
    in_str         данные передаваемые программе на страндартный вход.
    env_dict       словарь переменных окружения
    Возвращает (код возврата, список stdout+stderr)
    """
    pipe = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            env=env_dict,
                            close_fds=True,
                            shell=True)
    fout, fin, ferr = (pipe.stdout, pipe.stdin, pipe.stderr)
    # если есть данные на вход, передать их
    if in_str:
        fin.write(in_str)
    fin.close()
    res = map(lambda x: x.rstrip(), fout.readlines())
    fout.close()
    res += map(lambda x: x.rstrip(), ferr.readlines())
    ferr.close()
    # Код возврата
    retcode = pipe.wait()
    return retcode, res

def copyDir(srcDir, destDir):
    """Копируем директорию srcDir в destDir

    При копировании сохраняются владелец, группа, права
    """
    def ignoreFile(pathname, names):
        """Игнорирование сокетов при копировании"""
        ignore = []
        for name in names:
            if stat.S_ISSOCK(os.lstat(path.join(pathname, name))[stat.ST_MODE]):
                ignore.append(name)
        return ignore

    copytree(srcDir, destDir, ignore=ignoreFile)
    return True

def removeDir(rmDir):
        """Рекурсивное удаление директории"""
        rmtree(rmDir)
        return True

def getRunCommands(not_chroot=False):
    """List run program"""
    def getCmd(procNum):
        cmdLineFile = '/proc/%s/cmdline'%procNum
        try:
            if path.exists(cmdLineFile):
                if not_chroot:
                    root_link = '/proc/%s/root'%procNum
                    if os.readlink(root_link) != "/":
                        return ""
                return open(cmdLineFile,'r').read().strip()
        except Exception as e:
            pass
        return ""
    if not os.access('/proc',os.R_OK):
        return []
    return filter(None,
           map(getCmd,
           filter(lambda x:x.isdigit(),
           listDirectory('/proc'))))


def isMount(pathname):
    """В случае монтирования директории выдает другой примонтированный путь"""
    findPath = [path.abspath(pathname)]
    if findPath[0].startswith('/dev'):
        info = device.getUdevDeviceInfo(name=findPath[0])
        if 'DM_VG_NAME' in info and 'DM_LV_NAME' in info:
            lvmDeviceName = \
                '/dev/mapper/{vg}-{lv}'.format(vg=info['DM_VG_NAME'],
                                               lv=info['DM_LV_NAME'])
            findPath.append(lvmDeviceName)
    mtabFile = '/etc/mtab'
    if not os.access(mtabFile,os.R_OK):
        return ""
    return filter(lambda x: x!=findPath,
           reduce(lambda x,y: y,
           ifilter(lambda x: any(p in x for p in findPath),
           imap(lambda x: [x[0], x[1]],
           imap(lambda x: x.split(" "),
           open(mtabFile)))), [""]))[0]

def commonPath(*paths):
    """Return common path from list of paths"""
    paths = map(lambda x:path.normpath(x).split('/'),paths)
    res = map(lambda x:x[0],
        filter(lambda x:filter(lambda y:x[0]==y,x[1:]),zip(*paths)))
    return "/".join(res)

def childMounts(pathname):
    """Get all mount points which contain path"""
    if pathname != "none":
        absPath = path.abspath(pathname)
    else:
        absPath = pathname
    mtabFile = '/etc/mtab'
    if not os.access(mtabFile,os.R_OK):
        return ""
    return reduce(lambda x,y: x + [y],
                  filter(lambda x: commonPath(absPath,x[0])==absPath or \
                                   commonPath(absPath,x[1])==absPath,
                  map(lambda x: [x[0], x[1]],
                  map(lambda x: x.split(" "),
                  open(mtabFile)))),
                  [])


def pathJoin(*paths):
    """Складывает пути, в отличии от os.path.join, складывает абсолютные пути"""
    if len(paths)==1:
        return paths[0]
    return reduce(path.join,
                  filter(lambda x:x and x != "/",
                  map(lambda x: x.startswith("/") and x[1:] or x,
                  paths[1:])),paths[0])

def checkDigestFile(digestfile):
    """Check digest by digestfile"""
    reEntry = re.compile(r"# (\S+) HASH\n(\S+)  (\S+)",re.S)
    result = []
    for alg,hashdata,filename in \
        reEntry.findall(open(digestfile,'r').read()):
        if hasattr(hashlib,alg.lower()):
            hashobj = getattr(hashlib,alg.lower())
            filename = path.join(path.dirname(digestfile),filename)
            if os.path.exists(filename):
                digest = hashobj(open(filename,'r').read())
                result.append((alg,
                    digest.hexdigest().upper() == hashdata.upper()))
    return result

def listDirectory(directory,fullPath=False,onlyDir=False):
    """Get files from directory, if it exists"""
    if not path.exists(directory):
        return []
    try:
        if fullPath:
            if onlyDir:
                return filter(lambda x:path.isdir(x),
                       map(lambda x:path.join(directory,x),
                       os.listdir(directory)))
            else:
                return map(lambda x:path.join(directory,x),
                       os.listdir(directory))
        else:
            if onlyDir:
                return filter(lambda x:path.isdir(path.join(directory,x)),
                       os.listdir(directory))
            else:
                return os.listdir(directory)
    except OSError:
        pass
    return []

def makeDirectory(pathname):
    """Make directory and parent.

    Return False on error"""
    try:
        parent = path.split(path.normpath(pathname))[0]
        if not path.exists(parent):
            makeDirectory(parent)
        else:
            if path.exists(pathname):
                return True
        os.mkdir(pathname)
        return True
    except Exception, e:
        return False

def readLinesFile(filename):
    """Read file by line"""
    try:
        if path.exists(filename):
            for line in open(filename,'r'):
                yield line.rstrip('\n')
    except (OSError,IOError):
        pass
    finally:
        raise StopIteration

def readFile(filename,tailbyte=None):
    """
    Прочитать целый файл или вернуть пустую строку.
    tailbyte: прочитать только последнее указанное количество байт
    """
    try:
        if path.exists(filename):
            with open(filename,'r') as f:
                if tailbyte:
                    seeksize = max(0,os.stat(filename).st_size-tailbyte)
                    if seeksize:
                        f.seek(seeksize)
                return f.read()
    except (OSError,IOError):
        pass
    return ""

def writeFile(filename):
    """
    Открыть файл на запись и создать необходимые каталоги
    """
    dn = path.dirname(filename)
    if not path.exists(dn):
        os.makedirs(dn)
    return open(filename,'w')

import common

class getProgPath:
    """Get full path of program or False"""
    cache = {}

    def __call__(self,progname):
        baseprogname = path.basename(progname)
        if baseprogname in self.cache:
            return self.cache[baseprogname]
        env = {"LANG":"C"}
        env.update(os.environ.items() + [("PATH",common.getpathenv())] +\
                   env.items())
        res = runOsCommand("which %s"%baseprogname,env_dict=env)
        if path.isabs(progname) and path.exists(progname):
            self.cache[baseprogname] = progname
            return self.cache[baseprogname]
        elif res[0] == 0:
            self.cache[baseprogname] = res[1][0].strip()
            return self.cache[baseprogname]
        else:
            return False

getProgPath = getProgPath()

def checkUtils(*utils):
    """Check utils, exit if it not found and return fullpath"""
    retval = []
    for util in utils:
        utilPath = getProgPath(util)
        if not utilPath:
            raise FilesError(_("Command not found '%s'")%
                                     path.basename(util))
        retval.append(utilPath)
    if len(retval) == 1:
        return retval[0]
    else:
        return retval

def getCmdLineParam(paramName):
    """Get value of param /proc/cmdline. If param not found then empty.
    """
    cmdLine = '/proc/cmdline'
    paramName = "%s="%paramName
    params = \
        map(lambda x:x.partition('=')[2],
        filter(lambda x:x.startswith(paramName),
        readFile(cmdLine).split(' ')))
    if params:
        return params[-1]
    else:
        return ""

from device import getUUIDDict

def isFstabMount(pathname,mapDevUuid={},listFstab=[]):
    """Get mount point or device from fstab"""
    def removeQuotes(s):
        return s.replace('"','').replace("'","")
    if pathname == "swap":
        absPath = "swap"
    else:
        absPath = path.abspath(pathname)
    if not mapDevUuid:
        mapDevUuid.update(getUUIDDict())
    # convert fstab to
    # [['/dev/sda3', '/', '', 'reiserfs', 'noatime', '', '', '0', '2\n'],
    # ['/dev/sda5', '/var/calculate', 'reiserfs', 'noatime', '0', '0\n']]
    #if not listFstab:
    if not listFstab:
        listFstab.extend(
                map(lambda x: [mapDevUuid.get(removeQuotes(x[0]),x[0]),
                          x[1] if x[2] != "swap" else "swap"],
                filter(lambda x: len(x) >= 4,
                map(lambda x: filter(lambda x: x,
                              x.replace('\t',' ').split(' ')),
                filter(lambda x: not x.startswith('#') and x.strip(),
                open("/etc/fstab"))))))
    # get mount point or device or dir
    return filter(lambda x: x!=absPath,
           reduce(lambda x,y: y,
           filter(lambda x: absPath in x and x[1] != "none",
           listFstab),[""]))[0]

class SingletonParam(type):
    def __init__(cls, name, bases, dict):
        super(SingletonParam, cls).__init__(name, bases, dict)
        cls.instance = {}

    def __call__(cls,*args,**kw):
        keyarg = args[0] if args else ""
        if not keyarg in cls.instance:
            cls.instance[keyarg] = \
                super(SingletonParam, cls).__call__(*args, **kw)
        return cls.instance[keyarg]

import device

class FStab(object):
    """Data reader for fstab"""
    __metaclass__ = SingletonParam
    fstab_file = '/etc/fstab'
    NAME, DIR, TYPE, OPTS, FREQ, PASSNO = range(0,6)

    def __init__(self,fstab_file=None,devs=[]):
        if fstab_file:
            self.fstab_file = fstab_file
        self.cache = []
        self.rotateCache = []
        self.dictUUID = getUUIDDict(devs=devs)
        self.rebuildCache()

    def rebuildCache(self):
        """Rebuild cache from fstab file"""
        def setlen(ar):
            return ar[:6]+[""]*(6-len(ar))
        self.cache = \
            map(lambda x:setlen(map(lambda y:y.strip(),x.split())),
            filter(lambda x:x.strip() and not x.lstrip().startswith("#"),
            open(self.fstab_file,'r').read().split('\n')))
        for data in self.cache:
            convertDev = lambda x: path.realpath(x) if x.startswith('/') else x
            data[0] = device.getUdevDeviceInfo(
                       name=convertDev(self.dictUUID.get(data[0],data[0]))
                      ).get('DEVNAME',data[0])
            data[1] = data[1] if data[2] != "swap" else "swap"
        self.rotateCache = zip(*self.cache)

    def getBy(self,what=DIR,where=NAME,_in=None,eq=None,
              noteq=None,allentry=False):
        """Get data from fstab"""
        if not eq is None:
            filterfunc = lambda x: x[where] == eq
        elif not _in is None:
            filterfunc = lambda x: _in in x[where]
        else:
            filterfunc = lambda x: x[where] != noteq
        res = map(lambda x:x[what],filter(filterfunc,self.cache))
        if allentry:
            return res
        else:
            return "" if not res else res[-1]

    def getFields(self,*fields):
        """Get all data by specifie fields"""
        return zip(*reduce(lambda x,y:x+[self.rotateCache[y]],fields,[]))

    def isExists(self,what=DIR,eq=None,noteq=None):
        """Field with condition exist in fstab"""
        if not eq is None:
            filterfunc = lambda x: x[what] == eq
        else:
            filterfunc = lambda x: x[what] != noteq
        return bool(filter(filterfunc,self.cache))

def tarLinks(rootpath,archpath,skip=[]):
    """
    Поместить все симлинки в архив
    """
    links = []
    lenprefix = len(path.normpath(rootpath))+1
    if path.exists(archpath):
        os.unlink(archpath)
    # create arch
    tar = tarfile.open(archpath,"w:bz2")
    # find links
    if skip:
        reSkip = re.compile("|".join(map(lambda x:x.replace("*",".*"),
            skip))).search
    else:
        reSkip = lambda x:False
    for link in ifilterfalse(reSkip,
                find(rootpath,filetype="l",onefilesystem=True)):
        ti = tar.gettarinfo(link)
        ti.name = link[lenprefix:]
        tar.addfile(ti)
        links.append(link)
    tar.close()
    return links

def getch():
    """
    Get char from keyboard
    """
    import sys, tty, termios

    fd = sys.stdin.fileno()
    old_settings = termios.tcgetattr(fd)
    try:
        tty.setraw(sys.stdin.fileno())
        ch = sys.stdin.read(1)
    finally:
        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
    return ch

def find(directory,onefilesystem=False,filetype=None,
         fullpath=True, depth=None, downfilter=None):
    """
    Поиск файлов в каталоге
    """
    ldirectory = len(directory)
    if downfilter or onefilesystem or depth:
        _downfilter = lambda dp: (
            (depth is None or dp[ldirectory:].count('/') < depth) and
            (not downfilter or downfiler(dp)) and
            (not onefilesystem or not path.ismount(dp)))
    else:
        _downfilter = None

    if filetype:
        def dirfilter(dp):
            link = path.islink(dp)
            return ("l" in filetype and link) or ("d" in filetype and not link)
        _dirfilter = dirfilter
    else:
        _dirfilter = None

    if fullpath or _dirfilter or _downfilter:
        _dirfullpath = lambda root,fn: path.join(root,fn)
    else:
        _dirfullpath = lambda root,fn:fn

    if filetype:
        def filefilter(fp):
            link = path.islink(fp)
            return ("l" in filetype and link) or ("f" in filetype and not link)
        _filefilter = filefilter
    else:
        _filefilter = None

    if fullpath or _filefilter:
        _filefullpath = lambda root,fn: path.join(root,fn)
    else:
        _filefullpath = lambda root,fn:fn

    for root,directories,files in os.walk(directory):
        for fn in files:
            fp = _filefullpath(root,fn)
            if not _filefilter or _filefilter(fp):
                yield fp if fullpath else fn
        for dn in directories[:]:
            dp = _dirfullpath(root,dn)
            if not _dirfilter or _dirfilter(dp):
                yield dp if fullpath else dn
            if _downfilter and not _downfilter(dp):
                directories.remove(dn)

def copyWithPath(fn,dest,prefix=None):
    """
    Копировать файл в dst с созданием каталогов

    source: префикс исходного файла
    dest: каталог для копирывания
    fn: имя файла
    """
    try:
        if prefix:
            destFile = pathJoin(dest,fn[len(prefix):])
        else:
            destFile = pathJoin(dest,path.basename(fn))
        destDir = path.dirname(destFile)
        if not path.exists(destDir):
            os.makedirs(destDir)
        copy2(fn,destFile)
    except Exception as e:
        raise FilesError(
            _("Failed to copy '%(src)s' to '%(dst)s'")%
            {'src':fn,
             'dst':destFile})

def getLoopFromPath(directory):
    """
    Получить loop, использующие файлы в данной директории
    """
    losetup = getProgPath('losetup')
    p = process(losetup,'-a')
    rePattern = re.compile('(^\/dev\/loop[^:]+)[^(]+\(([^)]+)\).*')
    return map(lambda x:x[0],
            filter(lambda x:x[1].startswith(directory),
            reSearch(rePattern,
            p.readlines())))

def getMdRaidDevices():
    """
    Получить словарь: какой mdraid какие использует устройства
    """
    reMdInfo = re.compile('^(\w+)\s+:\s+active.*?raid\d+\s+(.*)')
    return dict(map(lambda x:(x[0],map(lambda x:x.partition('[')[0],
                            x[1].split())),
           reSearch(reMdInfo,readLinesFile('/proc/mdstat'))))

def sambaPasswordCheck(username,password,server,resource):
    """
    Подключиться к указанному samba ресурсу сервера по логину паролю
    """
    smbclient = getProgPath('/usr/sbin/smbclient')
    if not smbclient:
        raise FilesError(_("Command not found '%s'")%"smbclint")
    p = process(smbclient,"-U",username,"//%s/%s"%(server,resource),
                envdict={'PASSWD':password})
    return p.success()

class PercentProgress(processProgress):
    """
    Объект выдает прогресс, ища в выводе \d+%
    
    Args:
        part: количество прогрессов в программе
        delimeter: разделители строк по умолчанию \n и \r
        cachefilter: фильтр вывода программы (регулярная строка)
        startpart: используется для вывода процентов прогрессбар
                   с определенного места (0 по умолчанию)
        end:       конечный прогрессбар (по умолчанию)
        atty: для получения данных создается pty
    """
    def init(self,*args,**kwargs):
        self.rePerc = re.compile("(\d+)%",re.S)
        self.part = kwargs.get("part",1)
        if self.part < 1:
            self.part = 1
        self.add_offset = 100 / self.part
        self.offset = 0 + kwargs.get("startpart",0)*self.add_offset
        self.is_end = kwargs.get("end",True)
        self.stderr = STDOUT
        self.delimeter = re.compile("[%s]"%kwargs.get("delimeter","\n\r"))
        self.cachedata = re.compile(kwargs.get("cachefilter",
                "((?:\[31;01m\*|\[33;01m\*|error:|warning:|fatal:).*)"))
        self.atty = kwargs.get("atty",False)
        self.alldata = ""

    def _open(self):
        self.master,self.slave = None,None
        if self.atty:
            if not self.pipe:
                self.master,self.slave = os.openpty()
                self.pipe = Popen(self.command,
                                  stdout=self.slave,
                                  stderr=self.slave,
                                  cwd=self.cwd,
                                  close_fds=True,
                                  env=self.envdict)
                self.pipe.stdout = os.fdopen(self.master,'r')
        else:
            process._open(self)

    def processInit(self):
        self.percent = 0
        self.showval = 0
        if not self.offset:
            return 0

    def processEnd(self):
        if not self.slave is None:
            os.close(self.slave)
        if self.is_end:
            self.percent = 100
            return 100

    def processString(self, strdata):
        self.alldata += strdata
        match = self.rePerc.search(strdata)
        resSearch = self.cachedata.search(strdata)
        if resSearch:
            self.cacheresult.append(
                re.sub("\[31;01m\*|\[33;01m\*|\[.*?m",
                       "", resSearch.group(1)))
        if match:
            percent = int(match.group(1))
            if percent < self.percent:
                self.offset = self.offset + self.add_offset
            percent = percent / self.part
            if percent != self.percent:
                self.percent = percent
                showval = min(99,self.percent + self.offset)
                if showval != self.showval:
                    self.showval = showval
                    return self.showval


def set_active_tty(ttynum):
    """
    Установить активный терминал
    """
    chvt = getProgPath("/usr/bin/chvt2")
    if chvt:
        os.system('chvt %d &>/dev/null' % ttynum)