calculate-utils-3-lib/pym/calculate/lib/utils/files.py

# -*- coding: utf-8 -*-

# Copyright 2008-2017 Mir Calculate. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
import hashlib
import select
import random
import shutil
import string
from hashlib import sha256

import os
from os import path
import subprocess
from subprocess import Popen, PIPE, STDOUT
import stat
from shutil import copytree, rmtree, copy2
import re
import sys
from itertools import *
import tarfile
import tools
import fcntl
from tools import ignore
from glob import glob

from contextlib import contextmanager
import signal

_ = lambda x: x
from calculate.lib.cl_lang import setLocalTranslate

setLocalTranslate('cl_lib3', sys.modules[__name__])


@contextmanager
def timeout(seconds):
    def timeout_handler(signum, frame):
        pass

    original_handler = signal.signal(signal.SIGALRM, timeout_handler)

    try:
        signal.alarm(seconds)
        yield
    finally:
        signal.alarm(0)
        signal.signal(signal.SIGALRM, original_handler)


try:
    from magic import (open as type_file, MAGIC_NONE, MAGIC_CONTINUE,
                       MAGIC_MIME_TYPE, MAGIC_COMPRESS,
                       MAGIC_MIME_ENCODING, MAGIC_SYMLINK)
except ImportError as e:
    try:
        from magic import (open as type_file, NONE as MAGIC_NONE,
                           CONTINUE as MAGIC_CONTNUE,
                           MIME_TYPE as MAGIC_MIME_TYPE,
                           COMPRESS as MAGIC_COMPRESS,
                           MIME_ENCODING as MIME_ENCODING,
                           SYMLINK as MAGIC_SYMLINK)
    except (ImportError, AttributeError):
        type_file = None
        MAGIC_NONE = 0
        MAGIC_SYMLINK = 0x002
        MAGIC_COMPRESS = 0x004
        MAGIC_CONTINUE = 0x020
        MAGIC_MIME_TYPE = 0x010
        MAGIC_MIME_ENCODING = 0x400

from calculate.lib.cl_lang import setLocalTranslate

setLocalTranslate('cl_lib3', sys.modules[__name__])

reSearch = lambda pattern, listar: map(lambda x: x.groups(),
                                       filter(None,
                                              map(pattern.search, listar)))


class FilesError(Exception):
    pass


class StdoutableProcess(object):
    def __init__(self):
        self.pipe = None

    def get_stdout(self):
        return PIPE

    def close(self):
        pass


class KeyboardInputProcess(StdoutableProcess):
    def get_stdout(self):
        return None


class PipeProcess(StdoutableProcess):
    def get_stdout(self):
        return PIPE

class ProcessTimeout(FilesError):
    pass

class process(StdoutableProcess):
    """Execute system command by Popen
    
    Examples:
    
    execute program and get result:
    if process("/bin/gzip","/boot/somefile").success():
        print "Gzip success"

    unzip and process unzip data by cpio (list files):
    processGzip = process("/bin/gzip","-dc","/boot/initrd")
    processCpio = process("/bin/cpio","-tf",stdin=processGzip)
    filelist = processCpio.readlines()

    execute command and send data:
    processGrub = process("/sbin/grub")
    processGrub.write("root (hd0,0)\n")
    processGrub.write("setup (hd0)\n")
    processGrub.write("quit\n")
    isok = processGrub.success()

    union stdout and stderr:
    process("/bin/ls","/",stderr=STDOUT)
    
    result to stdout:
    process("/bin/ls",stdout=None)

    get data from keyboard:
    process("/bin/cat",stdin=None)
    """
    STDOUT = STDOUT
    PIPE = PIPE

    def __init__(self, command, *params, **kwarg):
        super(process, self).__init__()
        if "stdin" not in kwarg:
            stdin = PipeProcess()
        else:
            if kwarg["stdin"] is None:
                stdin = KeyboardInputProcess()
            else:
                stdin = kwarg["stdin"]
        self.stdout = kwarg.get("stdout", PIPE)
        self.envdict = kwarg.get("envdict", os.environ.copy())
        self.envdict["LANG"] = kwarg.get('lang', 'C')
        self.langc = "langc" in kwarg
        self.timeout = kwarg.get("timeout", None)

        self.stderr = kwarg.get("stderr", PIPE)
        self.cwd = kwarg.get("cwd", None)
        self.command = getProgPath(command)
        if not self.command:
            raise FilesError(_("Command not found '%s'") %
                             command)
        self.command = [self.command] + list(params)
        self.stdin = stdin
        self.iter = None
        self.cacheresult = None
        self.cacheerr = None
        self.communicated = False
        self._cachedata = []

    def _open(self):
        """Open pipe if it not open"""
        if not self.pipe:
            self.pipe = Popen(self.command,
                              stdout=self.stdout,
                              stdin=self.stdin.get_stdout(),
                              stderr=self.stderr,
                              cwd=self.cwd,
                              close_fds=True,
                              env=self.envdict)

    def get_stdout(self):
        """Get current stdout"""
        if self.pipe:
            self.close()
        self._open()
        self.cacheresult = ""
        return self.pipe.stdout

    def write(self, data):
        """Write to process stdin"""
        self._open()
        try:
            self.pipe.stdin.write(data)
            self.pipe.stdin.flush()
        except IOError as e:
            raise FilesError(str(e))

    def close(self):
        """Close stdin"""
        if self.pipe:
            self.pipe.poll()
            if self.pipe.stdin:
                self.pipe.stdin.close()
                self.pipe.stdin = None
        if self.stdin:
            self.stdin.close()
            self.stdin = None

    def readerr(self):
        self.read()
        if self.cacheerr is None:
            try:
                self.cacheerr = self.pipe.stderr.read()
            except IOError:
                self.cacheerr = ""
        return self.cacheerr

    def readByLine(self):
        _cacheerr = []
        try:
            self._open()
            if self.cacheresult is None:
                _stdout = self.pipe.stdout.fileno()
                _stderr = self.pipe.stderr.fileno()
                reads = [_stdout, _stderr]
                self.pipe.stdin.close()
                self.pipe.stdin = None
                while True:
                    ret = select.select(reads, [], [], self.timeout)
                    if not ret[0]:
                        self.kill()
                        raise ProcessTimeout()
                    for fd in ret[0]:
                        if fd == _stdout:
                            s = self.pipe.stdout.readline()
                            yield s
                            self._cachedata.append(s)
                        if fd == _stderr:
                            s = self.pipe.stderr.readline()
                            _cacheerr.append(s)
                    if self.pipe.poll() is not None:
                        s = " "
                        while True:
                            s = self.pipe.stdout.readline()
                            if not s:
                                break
                            yield s
                            self._cachedata.append(s)
                        while True:
                            s = self.pipe.stderr.readline()
                            if not s:
                                break
                            yield s
                            _cacheerr.append(s)
                        break
        except KeyboardInterrupt:
            self.kill()
            raise KeyboardInterrupt
        finally:
            if self.cacheresult is None:
                self.cacheresult = "\n".join(self._cachedata)
            self.cacheerr = "".join(_cacheerr)
            self.close()

    def read(self):
        """Read all data"""
        try:
            self._open()
            if self.cacheresult is None:
                if self.timeout is None:
                    self.cacheresult, self.cacheerr = self.pipe.communicate()
                else:
                    for line in self.readByLine():
                        pass
            return self.cacheresult
        except KeyboardInterrupt:
            self.kill()
            raise KeyboardInterrupt
        finally:
            self.close()

    def readlines(self):
        """Read lines"""
        return self.read().split('\n')

    def __iter__(self):
        """Get iterator"""
        if not self.iter:
            self.iter = iter(self.readlines())
        return self.iter

    def kill(self):
        """Kill this process"""
        if self.pipe:
            self.pipe.kill()

    def next(self):
        """Next string from stdout"""
        return self.__iter__().next()

    def returncode(self):
        """Get return code"""
        self.read()
        return self.pipe.returncode

    def success(self):
        """Success or not"""
        return self.returncode() == 0

    def failed(self):
        """Failed or not"""
        return self.returncode() != 0


def getModeFile(nameFile, mode="all"):
    """Выдает информацию о файле
    mode=="all"
        права файла, владелец, группа файла
    mode=="mode"
        права файла
    mode=="owner"
        владелец, группа файла
    """
    fst = os.lstat(nameFile)
    if mode == "all":
        return stat.S_IMODE(fst.st_mode), fst.st_uid, fst.st_gid
    if mode == "mode":
        return stat.S_IMODE(fst.st_mode)
    if mode == "owner":
        return fst.st_uid, fst.st_gid


def chown(fn, uid=None, gid=None):
    """
    Поменять uid и gid файла если текущие отличаются
    :param fn: файл
    :param uid: UID или None
    :param gid: GID или None
    """
    f_stat = os.stat(fn)
    if uid is None:
        uid = f_stat.st_uid
    if gid is None:
        gid = f_stat.st_gid
    if uid != f_stat.st_uid or gid != f_stat.st_gid:
        os.lchown(fn, uid, gid)


class FilePermission(object):
    """
    Права на файл
    """
    UserWrite = 0200
    UserRead = 0400
    UserExecute = 0100
    UserAll = UserWrite | UserRead | UserExecute

    GroupWrite = 020
    GroupRead = 040
    GroupExecute = 010
    GroupAll = GroupWrite | GroupRead | GroupExecute

    OtherWrite = 02
    OtherRead = 04
    OtherExecute = 01
    OtherAll = OtherWrite | OtherRead | OtherExecute

    SetUid = 04000
    SetGid = 02000
    StickyBit = 01000


def chmod(fn, mode):
    """
    Поменять права на файл если текущие права отличаются
    :param fn: файл
    :param mode: права (0755)
    """
    f_stat = os.stat(fn)
    if f_stat.st_mode != mode:
        os.chmod(fn, mode)


def chownR(directory, uid, gid):
    """Recusive chown"""

    def chownPaths(rootPath, listPath, uid, gid):
        for chPath in listPath:
            chownPath = path.join(rootPath, chPath)
            statInfo = os.lstat(chownPath)[stat.ST_MODE]
            if stat.S_ISLNK(statInfo):
                os.lchown(chownPath, uid, gid)
            else:
                os.chown(chownPath, uid, gid)

    for root, dirs, files in os.walk(directory):
        # меняем владельца директории
        os.chown(root, uid, gid)
        # Меняем владельца директорий
        chownPaths(root, dirs, uid, gid)
        # Меняем владельца файлов
        chownPaths(root, files, uid, gid)
    return True


class proxy_type_file:
    def __init__(self, flags):
        self.flags = flags

    def load(self):
        pass

    def setflags(self, flags):
        self.flags = flags

    def close(self):
        pass

    def _get_cmd_by_flags(self, flags):
        """
        Получить команду для выполнения по флагам
        """
        paramdict = {MAGIC_CONTINUE: "-k",
                     MAGIC_SYMLINK: "-L",
                     MAGIC_MIME_TYPE: "--mime-type",
                     MAGIC_MIME_ENCODING: "--mime-encoding",
                     MAGIC_COMPRESS: "-z"}
        appendParam = map(lambda x: paramdict[x],
                          filter(lambda x: flags & x,
                                 sorted(paramdict.keys())))
        fileCmd = getProgPath('/usr/bin/file')
        return [fileCmd, '-b'] + appendParam

    def file(self, filename):
        if path.exists(filename):
            processFile = process(
                *(self._get_cmd_by_flags(self.flags) + [filename]))
            if processFile.success():
                return processFile.read().rstrip()
        return None


class typeFile(object):
    """Получение типа файла"""

    def __init__(self, magic=MAGIC_MIME_ENCODING | MAGIC_MIME_TYPE):
        if type_file is None:
            self.magicObject = proxy_type_file(MAGIC_NONE)
        else:
            self.magicObject = type_file(MAGIC_NONE)
        self.magicObject.load()
        self.magicObject.setflags(magic)
        self.magic = magic

    def __del__(self):
        """Закрываем magic"""
        self.magicObject.close()

    def getMTypeBuf_(self, buf):
        try:
            ret = self.magicObject.buffer(buf)
        except UnicodeDecodeError:
            try:
                ret = self.magicObject.buffer(buf.decode('utf-8'))
            except UnicodeDecodeError:
                return None
        return ret

    def getMTypeBuf(self, buf):
        #buftype = typeFile(magic=0).getMTypeBuf
        self.magicObject.setflags(self.magic & (~(self.magic&0x4)))

        rdtype = self.getMTypeBuf_(buf)
        if not self.magic & 0x4:
            return rdtype

        if "LZ4" in rdtype:
            arch_cmd = '/usr/bin/lz4'
        elif "XZ" in rdtype:
            arch_cmd = '/usr/bin/xz'
        elif "bzip2" in rdtype:
            arch_cmd = '/usr/bin/bzip2'
        elif "gzip" in rdtype:
            arch_cmd = '/bin/gzip'
        elif "Zstandard" in rdtype:
            arch_cmd = '/usr/bin/zstd'
        else:
            return rdtype
            
        self.magicObject.setflags(self.magic)

        with open(os.devnull, 'w') as DEVNULL:
            gz = Popen([arch_cmd, "-dc"], stdout=PIPE, stderr=DEVNULL,
                        stdin=PIPE, close_fds=True)
            try:
                return "{} ({})".format(
                    self.getMTypeBuf_(gz.communicate(input=buf)[0]),
                    rdtype)
            finally:
                try:
                    gz.terminate()
                except OSError:
                    pass

    def getMType(self, filename):
        """Информация о типе файла"""
        try:
            ret = self.magicObject.file(filename)
        except UnicodeDecodeError:
            try:
                ret = self.magicObject.file(filename.decode('utf-8'))
            except UnicodeDecodeError:
                return None
        # fix for kernel 3.7.7 (bad work samba)
        if ret is None and self.magicObject.errno() == 5:
            r, w = os.pipe()
            devnull = os.open(os.devnull, os.O_WRONLY)
            cat = subprocess.Popen(['/bin/cat', filename], stdout=w,
                                   stderr=devnull, close_fds=True)
            ret = self.magicObject.descriptor(r)
            os.close(w)
            os.close(devnull)
            cat.wait()
        return ret

    def isBinary(self, filename):
        """является ли файл бинарным"""
        mime = self.getMType(filename)
        if mime:
            if mime.startswith("text"):
                return False
            else:
                return True
        return None


class processProgress(process):
    """
    Generator like process progress
    """

    def __init__(self, command, *params, **kwarg):
        process.__init__(self, command, *params, **kwarg)
        self.readsize = kwarg.get("readsize", 10)
        self.delimeter = re.compile("\n")
        self.init(**kwarg)
        self._cachedata = []
        self.buf = ""

    def init(self, *args, **kwarg):
        pass

    def processInit(self):
        pass

    def processEnd(self):
        pass

    def processString(self, strdata):
        self._cachedata.append(strdata)
        return strdata

    def processBuffer(self, buf):
        return ""

    def progress(self):
        try:
            res = self.processInit()
            if res is not None:
                yield res
            self._open()
            if self.cacheresult is None:
                self.buf = ""
                fd = self.pipe.stdout.fileno()
                fl = fcntl.fcntl(fd, fcntl.F_GETFL)
                fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
                hasData = True
                while hasData or self.pipe.poll() is None:
                    hasData = False
                    try:
                        part = self.pipe.stdout.read(self.readsize)
                        if not part:
                            continue
                    except IOError as er:
                        # re-raise not "Resource temporary unavailable"
                        if er.errno != 11:
                            raise
                        continue
                    hasData = True
                    if self.buf:
                        self.buf += part
                    else:
                        self.buf = part
                    res = self.processBuffer(part)
                    if res:
                        yield res
                    while self.delimeter.search(self.buf):
                        strdata, self.buf = self.delimeter.split(self.buf, 1)
                        res = self.processString(strdata)
                        if res:
                            yield res
            res = self.processEnd()
            if res is not None:
                yield res
        except KeyboardInterrupt:
            self.pipe.kill()
            raise
        finally:
            if self._cachedata:
                self.cacheresult = "\n".join(self._cachedata)
            else:
                self.cacheresult = ""


def countFiles(dirpath, onefilesystem=True):
    """
    Count files in specified dirpath
    """
    num = 1
    for dirpath, dirnames, filenames in os.walk(dirpath):
        num += len(set(dirnames) | set(filenames))
        if onefilesystem:
            mountDirs = filter(lambda x: path.ismount(path.join(dirpath, x)),
                               dirnames)
            for dirname in mountDirs:
                dirnames.remove(dirname)
    return num


def getFilesCount(directory):
    """Alias for compatibility"""
    return countFiles(directory, onefilesystem=False)


def copyDir(srcDir, destDir):
    """Копируем директорию srcDir в destDir

    При копировании сохраняются владелец, группа, права
    """

    def ignoreFile(pathname, names):
        """Игнорирование сокетов при копировании"""
        ignore = []
        for name in names:
            if stat.S_ISSOCK(os.lstat(path.join(pathname, name))[stat.ST_MODE]):
                ignore.append(name)
        return ignore

    copytree(srcDir, destDir, ignore=ignoreFile)
    return True


def removeDir(rmDir):
    """Рекурсивное удаление директории"""
    rmtree(rmDir)
    return True

def clearDirectory(dn):
    """
    Очистить содержимое директории
    :param dn:
    :return:
    """
    for dn in listDirectory(dn, fullPath=True):
        removeDir(dn)

def check_rw(dn):
    """
    Проверить досупена ли указанная директория на запись
    :param dn:
    :return:
    """
    def _random_string(l=10):
        return "".join(
            random.choice(string.ascii_letters) for i in xrange(0, l))

    mark_file = None
    while mark_file is None or path.exists(mark_file):
        mark_file = path.join(dn, "_mark_%s" % _random_string())
    try:
        open(mark_file, 'w').close()
        os.unlink(mark_file)
        return True
    except (OSError, IOError):
        return False


def quite_unlink(fn):
    """
    Проверить существует ли файл и удалить его, не поднимать Exception
    если не удалось удалить
    :param fn:
    :return:
    """
    with ignore(OSError):
        if path.lexists(fn):
            os.unlink(fn)

def removeFileWithEmptyDirectory(rmName, stopDirectory="/"):
    """
    Удалить файл и все пустые родительские директории
    :param rmName: путь до файла или директории
    """
    if rmName != stopDirectory and path.exists(rmName):
        if path.isdir(rmName):
            if not listDirectory(rmName, fullPath=True):
                os.rmdir(rmName)
                removeFileWithEmptyDirectory(path.dirname(rmName),
                                             stopDirectory=stopDirectory)
        else:
            os.unlink(rmName)
            removeFileWithEmptyDirectory(path.dirname(rmName),
                                         stopDirectory=stopDirectory)


def pathJoin(*paths):
    """Складывает пути, в отличии от os.path.join, складывает абсолютные пути"""
    if len(paths) == 1:
        return paths[0]
    return reduce(path.join,
                  filter(lambda x: x and x != "/",
                         map(lambda x: x.startswith("/") and x[1:] or x,
                             paths[1:])), paths[0])


def listDirectory(directory, fullPath=False, onlyDir=False):
    """Get files from directory, if it exists"""
    if not path.exists(directory):
        return []
    try:
        if fullPath:
            if onlyDir:
                return filter(lambda x: path.isdir(x),
                              map(lambda x: path.join(directory, x),
                                  os.listdir(directory)))
            else:
                return map(lambda x: path.join(directory, x),
                           os.listdir(directory))
        else:
            if onlyDir:
                return filter(lambda x: path.isdir(path.join(directory, x)),
                              os.listdir(directory))
            else:
                return os.listdir(directory)
    except OSError:
        pass
    return []


def makeDirectory(pathname, force=False):
    """
    Создать каталог вместе с вышестоящими. force - используется для удаления
    файла, который является частью пути.

    Возвращает False при ошибке"""
    try:
        parent = path.split(path.normpath(pathname))[0]
        if not path.exists(parent):
            makeDirectory(parent)
        else:
            if path.exists(pathname):
                if force and not path.isdir(pathname):
                    os.unlink(pathname)
                else:
                    return True
        os.mkdir(pathname)
        return True
    except (OSError, IOError):
        return False


def readLinesFile(filename, grab=False):
    """Read file by line"""
    try:
        if path.exists(filename):
            for line in open(filename, 'r'):
                if grab:
                    line = line.strip()
                    if not line.startswith("#"):
                        yield line
                else:
                    yield line.rstrip('\n')
    except (OSError, IOError):
        pass
    finally:
        raise StopIteration


def isEmptyFile(filename, grab=False):
    return not any(readLinesFile(filename, grab=grab))


def grepFile(filename, regexp, flags=0):
    """
    Получить из файла данные по регулярному выражению
    """
    data = readFile(filename)
    match = re.search(regexp, data, flags=flags)
    if match:
        return match.group()
    return ""

def readFile(filename):
    """
    Прочитать целый файл или вернуть пустую строку в случае ошибки
    """
    try:
        if path.exists(filename):
            with open(filename, 'r') as f:
                return f.read()
    except (OSError, IOError) as e:
        mod, lno = tools.get_traceback_caller(*sys.exc_info())
        sys.stderr.write("WARNING: file read error, {}({}:{})\n".format(
            str(e), mod, lno))
        sys.stderr.flush()
    return ""

def readFileEx(filename, tailbyte=None, headbyte=None, grab=False):
    """
    Прочитать целый файл или вернуть пустую строку.
    tailbyte: прочитать только последнее указанное количество байт
    headbyte: прочитать только первое указанное количество байт
    grab: отбросить пустые строки и комментарии (#)
    """
    try:
        if path.exists(filename):
            with open(filename, 'r') as f:
                if grab:
                    filterfunc = lambda s: "\n".join(
                        x for x in s.split("\n")
                        if not x.startswith("#") and x.strip())
                else:
                    filterfunc = lambda x: x
                if tailbyte:
                    seeksize = max(0, os.stat(filename).st_size - tailbyte)
                    if seeksize:
                        f.seek(seeksize)
                if headbyte:
                    return filterfunc(f.read(headbyte))
                else:
                    return filterfunc(f.read())
    except (OSError, IOError):
        mod, lno = tools.get_traceback_caller(*sys.exc_info())
        sys.stderr.write("WARNING: file read error, {}({}:{})\n".format(
            str(e), mod, lno))
        sys.stderr.flush()
    return ""


def writeFile(filename):
    """
    Открыть файл на запись и создать необходимые каталоги
    """
    dn = path.dirname(filename)
    if not path.exists(dn):
        os.makedirs(dn)
    return open(filename, 'w')


class GetProgPath(object):
    """Get full path of program or False"""
    cache = {}

    def __call__(self, progname, prefix="/"):
        baseprogname = path.basename(progname)
        key = (baseprogname, prefix)
        if key in self.cache:
            return self.cache[key]
        bindir = ('/bin', '/sbin', '/usr/sbin', '/usr/bin')
        if progname.startswith('/'):
            if path.exists(pathJoin(prefix, progname)):
                self.cache[key] = progname
                return progname
        for progname in (pathJoin(x, baseprogname) for x in bindir):
            if path.exists(pathJoin(prefix, progname)):
                self.cache[key] = progname
                return progname
        return False
        # env = {"LANG":"C"}
        # env.update(os.environ.items() + [("PATH",common.getpathenv())] +\
        #           env.items())
        # res = runOsCommand("which %s"%baseprogname,env_dict=env)
        # if path.isabs(progname) and path.exists(progname):
        #    self.cache[baseprogname] = progname
        #    return self.cache[baseprogname]
        # elif res[0] == 0:
        #    self.cache[baseprogname] = res[1][0].strip()
        #    return self.cache[baseprogname]
        # else:
        #    return False


getProgPath = GetProgPath()

def checkUtils(*utils):
    """Check utils, exit if it not found and return fullpath"""
    retval = []
    for util in utils:
        utilPath = getProgPath(util)
        if not utilPath:
            raise FilesError(_("Command not found '%s'") %
                             path.basename(util))
        retval.append(utilPath)
    if len(retval) == 1:
        return retval[0]
    else:
        return retval


def getCmdLineParam(paramName):
    """Get value of param /proc/cmdline. If param not found then empty.
    """
    cmdLine = '/proc/cmdline'
    paramName = "%s=" % paramName
    params = \
        map(lambda x: x.partition('=')[2],
            filter(lambda x: x.startswith(paramName),
                   readFile(cmdLine).split(' ')))
    if params:
        return params[-1]
    else:
        return ""


def tarLinks(rootpath, archpath, skip=()):
    """
    Поместить все симлинки в архив
    """
    links = []
    lenprefix = len(path.normpath(rootpath)) + 1
    if path.exists(archpath):
        os.unlink(archpath)
    # create arch
    tar = tarfile.open(archpath, "w:bz2")
    # find links
    if skip:
        reSkip = re.compile("|".join(map(lambda x: x.replace("*", ".*"),
                                         skip))).search
    else:
        reSkip = lambda x: False
    for link in ifilterfalse(reSkip,
                             find(rootpath, filetype=FindFileType.SymbolicLink,
                                  onefilesystem=True)):
        ti = tar.gettarinfo(link)
        ti.name = link[lenprefix:]
        tar.addfile(ti)
        links.append(link)
    tar.close()
    return links


def getch():
    """
    Get char from keyboard
    """
    import sys, tty, termios

    try:
        fd = sys.stdin.fileno()
        old_settings = termios.tcgetattr(fd)
        try:
            tty.setraw(sys.stdin.fileno())
            ch = sys.stdin.read(1)
        finally:
            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
        return ch
    except Exception:
        return ""


class FindFileType(object):
    """
    Типы файлов для find
    """
    RegularFile = "f"
    Directory = "d"
    SymbolicLink = "l"
    Block = "b"
    Character = "c"
    NamedPipe = "p"


def find(directory, onefilesystem=False, filetype=None,
         fullpath=True, depth=None, downfilter=None):
    """
    Поиск файлов в каталоге
    :param directory: каталог в котором производится поиск
    :param onefilesystem: искать только в текущей файловой системе
    :param filetype: тип файла (FindFileType)
    :param fullpath: выдавать полный путь
    :param depth: глубина поиска
    :param downfilter: фильтр поиска
    :return: генератор списка файлов
    """
    directory = path.normpath(directory)
    ldirectory = len(directory)
    if downfilter or onefilesystem or depth:
        _downfilter = lambda dp: (
            (depth is None or dp[ldirectory:].count('/') < depth) and
            (not downfilter or downfilter(dp)) and
            (not onefilesystem or not path.ismount(dp)))
    else:
        _downfilter = None

    if filetype:
        def dirfilter(dp):
            link = path.islink(dp)
            return ((FindFileType.SymbolicLink in filetype and link)
                    or (FindFileType.Directory in filetype and not link))

        _dirfilter = dirfilter
    else:
        _dirfilter = None

    _dirfullpath = lambda root, fn: path.join(root, fn)

    if filetype:
        def filefilter(fp):
            link = path.islink(fp)
            return ((FindFileType.SymbolicLink in filetype and link)
                    or (FindFileType.RegularFile in filetype and not link))

        _filefilter = filefilter
    else:
        _filefilter = None

    _filefullpath = lambda root, fn: path.join(root, fn)

    if directory.endswith('/'):
        clearlen = len(directory)
    else:
        clearlen = len(directory) + 1
    shortpath = lambda fp: fp[clearlen:]

    for root, directories, files in os.walk(directory):
        for fn in files:
            fp = _filefullpath(root, fn)
            if not _filefilter or _filefilter(fp):
                yield fp if fullpath else shortpath(fp)
        for dn in directories[:]:
            dp = _dirfullpath(root, dn)
            if not _dirfilter or _dirfilter(dp):
                yield dp if fullpath else shortpath(dp)
            if _downfilter and not _downfilter(dp):
                directories.remove(dn)


def isEmpty(dn):
    """
    Проверить пустая ли директория
    :param dn: директория
    :return:
    """
    return not listDirectory(dn)


def copyWithPath(fn, dest, prefix=None):
    """
    Копировать файл в dst с созданием каталогов

    dest: каталог для копирования
    fn: имя файла
    prefix: префикс исходного файла

    Пример:
    copyWithPath("/etc/conf.d/net", "/tmp/conf", prefix="/etc")
    /etc/conf.d/net -> /tmp/conf/conf.d/net
    """
    if prefix:
        destFile = pathJoin(dest, fn[len(prefix):])
    else:
        destFile = pathJoin(dest, path.basename(fn))
    try:
        destDir = path.dirname(destFile)
        if not path.exists(destDir):
            os.makedirs(destDir)
        copy2(fn, destFile)
    except Exception:
        raise FilesError(
            _("Failed to copy '%(src)s' to '%(dst)s'") %
            {'src': fn,
             'dst': destFile})


def getLoopFromPath(directory):
    """
    Получить loop, использующие файлы в данной директории
    """
    losetup = getProgPath('losetup')
    p = process(losetup, '-a')
    rePattern = re.compile('(^/dev/loop[^:]+)[^(]+\(([^)]+)\).*')
    return map(lambda x: x[0],
               filter(lambda x: x[1].startswith(directory),
                      reSearch(rePattern,
                               p.readlines())))


def getMdRaidDevices():
    """
    Получить словарь: какой mdraid какие использует устройства
    """
    reMdInfo = re.compile('^(\w+)\s+:\s+active.*?raid\d+\s+(.*)')
    return dict(map(lambda x: (x[0], map(lambda x: x.partition('[')[0],
                                         x[1].split())),
                    reSearch(reMdInfo, readLinesFile('/proc/mdstat'))))


class PercentProgress(processProgress):
    """
    Объект выдает прогресс, ища в выводе \d+%
    
    Args:
        part: количество прогрессов в программе
        delimeter: разделители строк по умолчанию \n и \r
        cachefilter: фильтр вывода программы (регулярная строка)
        startpart: используется для вывода процентов прогрессбар
                   с определенного места (0 по умолчанию)
        end:       конечный прогрессбар (по умолчанию)
        atty: для получения данных создается pty
    """

    def init(self, *args, **kwargs):
        self.rePerc = re.compile("(\d+(?:\.\d+)?)%", re.S)
        self.part = kwargs.get("part", 1)
        if self.part < 1:
            self.part = 1
        self.add_offset = 100 / self.part
        self.offset = 0 + kwargs.get("startpart", 0) * self.add_offset
        self.is_end = kwargs.get("end", True)
        self.stderr = STDOUT
        self.delimeter = re.compile("[%s]" % kwargs.get("delimeter", "\n\r"))
        self.cachedata = re.compile(kwargs.get("cachefilter",
                                               "((?:\[31;01m\*|\[33;01m\*|"
                                               "Bad Option|No space left|FATAL ERROR|SYNTAX:|mkisofs:|"
                                               "error:|warning:|fatal:).*)"))
        self.atty = kwargs.get("atty", False)
        self.alldata = ""

    def _open(self):
        self.master, self.slave = None, None
        if self.atty:
            if not self.pipe:
                self.master, self.slave = os.openpty()
                import termios, struct, fcntl

                TIOCSWINSZ = getattr(termios, 'TIOCSWINSZ', -2146929561)
                s = struct.pack('HHHH', 25, 80, 0, 0)
                fcntl.ioctl(self.slave, TIOCSWINSZ, s)
                self.pipe = Popen(self.command,
                                  stdout=self.slave,
                                  stderr=self.slave,
                                  cwd=self.cwd,
                                  close_fds=True,
                                  env=self.envdict)
                self.pipe.stdout = os.fdopen(self.master, 'r')
        else:
            process._open(self)

    def processInit(self):
        self.percent = 0
        self.showval = 0
        if not self.offset:
            return 0

    def processEnd(self):
        if not self.slave is None:
            os.close(self.slave)
        if self.is_end:
            self.percent = 100
            return 100

    def processString(self, strdata):
        self.alldata += strdata
        match = self.rePerc.search(strdata)
        resSearch = self.cachedata.search(strdata)
        if resSearch:
            self._cachedata.append(
                re.sub("\[31;01m\*|\[33;01m\*|\[.*?m",
                       "", resSearch.group(1)))
        if match:
            percent = int(float(match.group(1)))
            if percent < self.percent:
                self.offset += self.add_offset
            percent /= self.part
            if percent != self.percent:
                self.percent = percent
                showval = min(99, self.percent + self.offset)
                if showval != self.showval:
                    self.showval = showval
                    return self.showval


def set_active_tty(ttynum):
    """
    Установить активный терминал
    """
    chvt = getProgPath("/usr/bin/chvt")
    if chvt:
        os.system('chvt %d &>/dev/null' % ttynum)


def get_active_tty():
    """
    Получить активный терминал
    """
    getvt = getProgPath("/usr/bin/fgconsole")
    if getvt:
        p = process(getvt)
        if p.success():
            return p.read().strip()
    return None


class scanDirectory(object):
    """Класс для cканирования директории"""

    def processingFile(self, pathname, prefix):
        """Обработка в случае файла"""
        return True

    def processingDirectory(self, pathname, prefix):
        """Обработка в случае директории если возвращаем None то пропуск дир."""
        return True

    def scanningDirectory(self, scanDir, skipFile=(), skipDir=(),
                          prefix=None, flagDir=False):
        """Сканирование и обработка шаблонов в директории scanDir"""
        ret = True
        if not prefix:
            prefix = path.join(scanDir, "")[:-1]
        if not flagDir:
            # проверка корневой директории
            retDir = self.processingDirectory(scanDir, scanDir)
            if retDir is None:
                return None
            elif retDir is False:
                return False
        if flagDir or stat.S_ISDIR(os.lstat(scanDir)[stat.ST_MODE]):
            for absPath in sorted(listDirectory(scanDir, fullPath=True)):
                relPath = absPath.split(prefix)[1]
                stInfo = os.lstat(absPath)
                statInfo = stInfo[stat.ST_MODE]
                if stat.S_ISREG(statInfo):
                    # Обработка файла
                    if relPath in skipFile:
                        continue
                    if not self.processingFile(absPath, prefix):
                        return False
                elif stat.S_ISDIR(statInfo):
                    # Обработка директории
                    if relPath in skipDir:
                        continue
                    retDir = self.processingDirectory(absPath, prefix)
                    if retDir is None:
                        continue
                    elif retDir is False:
                        return False
                    ret = self.scanningDirectory(absPath, skipFile,
                                                 skipDir, prefix, True)
                    if ret is False:
                        return False
        return ret


def calculate_exclude(base_dn, exclude=(), include=()):
    """
    Вычисление исключений с использованием фильтров include
    """
    # корректируем входные данные, чтобы не учитывался первый /
    exclude = [x[1:] if x.startswith('/') else x for x in exclude]
    include = [x[1:] if x.startswith('/') else x for x in include]
    # перебираем все исключения
    for exclude_dn in exclude:
        # ищем фильтры для текущего исключения
        f_include = [x for x in include
                     if x == exclude_dn or x.startswith("%s/" % exclude_dn)]
        # если фильтра нет - то исключение добавляется полностью
        if not f_include:
            yield exclude_dn
        else:
            # пребираем все файлы в указанно исключении
            for root, dirs, files in os.walk(path.join(base_dn, exclude_dn)):
                rel_root = root[len(base_dn) + 1:]
                # если файл указан в фильтре, пропускаем его
                for fn in (path.join(rel_root, x) for x in files):
                    if fn not in f_include:
                        yield fn
                filtered_dn = []
                for dn in dirs:
                    rel_dn = path.join(rel_root, dn)
                    # если директория не используется как часть фильтра - возвращем её
                    if all(not x.startswith("%s/" % rel_dn) and x != rel_dn
                           for x in f_include):
                        filtered_dn.append(dn)
                        yield rel_dn
                    elif rel_dn in f_include:
                        filtered_dn.append(dn)

                # удаляем из обрабатываемых дирекоторий не использующиеся в фильтре
                # или указанную конечную директорию
                for dn in filtered_dn:
                    dirs.remove(dn)


def get_free_dirname(dn):
    """
    Возвращает имя указанной директорий если оно отсутствует либо добавляются
    цифры
    :param dn:
    :return:
    """
    if not path.exists(dn):
        return dn
    base_dn, name = path.split(dn)
    for n in range(0, 9999):
        dn = path.join(base_dn, "%s.%02d" % (name, n))
        if not path.exists(dn):
            return dn
    raise FilesError(_("No directory name available for %s") % dn)


def dir_sync(src, dst):
    """
    Скопировать файлы из src в dst, заменяя одинаковые файлы
    :return:
    """
    if not path.isdir(src):
        raise FilesError(_("%s is not directory") % src)
    if path.lexists(dst):
        dst_stat = os.lstat(dst)
        if stat.S_ISDIR(dst_stat.st_mode):
            for fn in listDirectory(src):
                src_fn = path.join(src, fn)
                dst_fn = path.join(dst, fn)
                fn_stat = os.lstat(src_fn)
                if stat.S_ISDIR(fn_stat.st_mode):
                    dir_sync(src_fn, dst_fn)
                    continue
                if path.lexists(dst_fn):
                    dst_stat = os.lstat(dst_fn)
                    if stat.S_ISDIR(dst_stat.st_mode):
                        shutil.rmtree(dst_fn)
                    else:
                        os.unlink(dst_fn)
                shutil.copy2(src_fn, dst_fn)
            return
        else:
            os.unlink(dst)
    shutil.copytree(src, dst)


def tar_directory(dn, archfile):
    """
    Заархивировать указанную директорию
    :param dn:
    :param archfile:
    :return:
    """
    try:
        with tarfile.open(archfile, "w:bz2") as tar:
            for x in listDirectory(dn, fullPath=True):
                tar.add(x, path.basename(x))
    except (IOError, OSError) as e:
        raise FilesError(_("Failed to create tarball: %s") % str(e))

def tar_xz_directory(dn, archfile):
    """
    Заархивировать указанную директорю в xz формате
    :param dn:
    :param archfile:
    :return:
    """
    try:
        tar = "/bin/tar"
        p = process(tar, "-cJf", archfile, "-C", dn, *listDirectory(dn))
        if p.failed():
            raise FilesError(_("Failed to create tarball: %s")
                             % str(p.readerr()))
    except (IOError, OSError) as e:
        raise FilesError(_("Failed to create tarball: %s") % str(e))

def tar_list(file_list, archfile, prefix='/'):
    """
    Заархивировать указанный список
    :param file_list:
    :param archfile:
    :param prefix:
    :return:
    """
    try:
        with tarfile.open(archfile, "w:bz2") as tar:
            for fn in (pathJoin(prefix, x) for x in file_list):
                tar.add(fn, path.relpath(fn, prefix))
    except (IOError, OSError) as e:
        raise FilesError(_("Failed to create tarball: %s") % str(e))


def tar_unpack(archfile, target_dn):
    """
    Распаковать архив в указанную директорию
    :param archfile:
    :param target_dn:
    :return:
    """
    try:
        with tarfile.open(archfile, "r:bz2") as tar:
            for item in tar:
                tar.extract(item, target_dn)
                os.utime(path.join(target_dn, item.path),
                         (item.mtime, item.mtime))
    except (tarfile.TarError, IOError, OSError) as e:
        raise FilesError(_("Failed to create tarball: %s") % str(e))


class RsyncOptions(object):
    Xattrs = "--xattrs"
    KeepDirLinks = "--keep-dirlinks"
    OneFileSystem = "--one-file-system"
    Archive = "--archive"
    FilesFromStdin = "--files-from=-"

    @classmethod
    def Chown(cls, uid, gid):
        return "--chown={0}:{1}".format(uid, gid)

    # опции используемые при "скрывании пакета"
    # --keep-dirlinks решает проблему восстановления "скрытых" файлов
    HidePackageOpts = (Xattrs, KeepDirLinks, OneFileSystem, Archive)

def rsync_files(source_dn, target_dn, *file_list, **kw):
    """
    Перенести файлы file_list из prefix в target_dn с сохранением путей и
    аттрибутов
    :param file_list:
    :param target_dn:
    :param prefix:
    :return:
    """
    opts = kw.get('opts', ())
    rsync = getProgPath('/usr/bin/rsync')
    source_dn = "%s/" % path.normpath(source_dn)
    target_dn = "%s/" % path.normpath(target_dn)
    if file_list:
        params = list(opts) + [RsyncOptions.FilesFromStdin, source_dn,
                               target_dn]
    else:
        params = list(opts) + [source_dn, target_dn]
    p = process(rsync, *params, stderr=PIPE)
    if file_list:
        for fn in file_list:
            p.write("%s\n" % fn)
    if p.failed():
        raise FilesError(_("Failed to rsync {src} to {dst}: {error}").format(
            src=source_dn, dst=target_dn,
            error=p.readerr()))

def xz(data, decompress=False):
    xz_prog = getProgPath('/usr/bin/xz')
    p = Popen([xz_prog, "-dc" if decompress else "-zc"],
              stdin=PIPE, stderr=PIPE, stdout=PIPE, close_fds=True)
    data, errors = p.communicate(input=data)
    p.poll()
    if p.returncode == 0:
        return data
    else:
        raise FilesError(errors)

class XZStreamRead(object):
    """
    XZ объект для чтения данных для tarfile
    """
    def __init__(self, data):
        self._data = xz(data, decompress=True)
        self.pos = 0

    def tell(self):
        return self.pos

    def seek(self, n=0):
        self.pos = n

    def read(self, size=None):
        if size is None:
            try:
                return self._data[self.pos:]
            finally:
                self.pos = len(self._data)
        try:
            return self._data[self.pos:self.pos+size]
        finally:
            self.pos += min(size,len(self._data))

@contextmanager
def xztaropen(fn):
    f = None
    try:
        f = tarfile.TarFile(mode="r", fileobj=XZStreamRead(readFile(fn)))
        yield f
    finally:
        if f:
            f.close()

def sha256sum(*fns):
    """
    Вычислить контрольную сумму у перечисленных файлов
    :param fns:
    :return:
    """
    kb = 1024
    hasher = sha256()
    for fn in fns:
        with open(fn) as f:
            b = True
            while b:
                b = f.read(512 * kb)
                hasher.update(b)
    return hasher.hexdigest()

def forceFileRename(source_fn, target_fn):
    """
    Принудительно перенести файл, даже если назначение занято
    :param source_fn:
    :param target_fn:
    :return:
    """
    if path.exists(target_fn) and not path.isfile(target_fn):
        if path.isdir(target_fn):
            removeDir(target_fn)
        else:
            os.unlink(target_fn)
    os.rename(source_fn, target_fn)


def move_files(source, target):
    """
    Переменстить все файлы из source в target вместе со структурой
    :param source:
    :param target:
    :return:
    """
    for fn in find(source, filetype=FindFileType.RegularFile,
                   fullpath=False):
        source_fn = path.join(source, fn)
        target_fn = path.join(target, fn)
        target_dn = path.dirname(target_fn)
        # принудительно создать каталог, удалив файлы являющиейся частью пути
        makeDirectory(target_dn, force=True)
        forceFileRename(source_fn, target_fn)
    clearDirectory(source)

class RCSError(Exception):
    pass

class DirectoryRCS(object):
    """
    Объект оперирует тремя каталогами, в котором хранит:
    фиксированную версию данных
    изменения для фиксирования данных (без учёта удаления файлов)
    рабочая версия данных
    """
    def __init__(self, workdir, indexdir, fixeddir):
        self.workdir = workdir
        self.indexdir = indexdir
        self.fixeddir = fixeddir

    def is_clear(self):
        """
        Изменения есть только в workdir или в fixed
        :return:
        """
        return (isEmpty(self.indexdir) and isEmpty(self.workdir) and
                not isEmpty(self.fixeddir))

    def is_worked(self):
        """
        Есть непроиндексированные изменения
        :return:
        """
        return not isEmpty(self.workdir)

    def not_prepared(self):
        return (isEmpty(self.indexdir) and not isEmpty(self.workdir) and
                isEmpty(self.fixeddir))

    def fixing(self):
        """
        Зафиксировать все изменения
        :return:
        """
        for dn in (self.indexdir, self.workdir):
            if not isEmpty(dn):
                self.movedata(dn, self.fixeddir)

            if path.exists(dn):
                removeDir(dn)

    def indexing(self):
        """
        Зафиксировать рабочие изменения в индекс
        :return:
        """
        self.movedata(self.workdir, self.indexdir)

    def movedata(self, source, target):
        """
        Поместить все work изменения в index
        :return:
        """
        if not path.exists(target) and path.exists(source):
            try:
                os.rename(source, target)
            except OSError as e:
                raise RCSError(_("Failed to rename data: %s") % str(e))
            makeDirectory(source)
        else:
            try:
                move_files(source, target)
            except FilesError as e:
                raise RCSError(_("Failed to move data: %s") % str(e))


class RealFs(tools.GenericFs):
    """
    Файловая система, предоставляющая доступ к реальным данным
    """
    def __init__(self, prefix="/"):
        self.prefix = prefix
        if prefix == "/":
            self.remove_prefix = lambda x: x
        else:
            self.remove_prefix = self._remove_prefix

    def _remove_prefix(self, dn):
        return dn[:len(self.prefix)]

    def _path(self, dn):
        return pathJoin(self.prefix, dn)

    def exists(self, fn):
        return path.lexists(self._path(fn))

    def read(self, fn):
        return readFile(self._path(fn))

    def glob(self, dn):
        for fn in glob(self._path(dn)):
            yield self.remove_prefix(fn)

    def realpath(self, fn):
        return self.remove_prefix(path.realpath(fn))

    def write(self, fn, data):
        with writeFile(fn) as f:
            f.write(data)

    def listdir(self, dn, fullpath=False):
        if fullpath:
            return [self.remove_prefix(x)
                    for x in listDirectory(dn, fullPath=True)]
        else:
            return listDirectory(dn, fullPath=False)

def checkDigestFile(digestfile):
    """Check digest by digestfile"""
    reEntry = re.compile(r"# (\S+) HASH\n(\S+)  (\S+)", re.S)
    result = []
    for alg, hashdata, filename in reEntry.findall(readFile(digestfile)):
        if hasattr(hashlib, alg.lower()):
            hashobj = getattr(hashlib, alg.lower())
            filename = path.join(path.dirname(digestfile), filename)
            if os.path.exists(filename):
                with open(filename, 'r') as f:
                    digest = hashobj(f.read())
                result.append((alg,
                               digest.hexdigest().upper() == hashdata.upper()))
    return result


def getRunCommands(not_chroot=False, chroot=None, uid=None, withpid=False):
    """List run program"""

    def getCmd(procNum):
        cmdLineFile = '/proc/%s/cmdline' % procNum
        try:
            if uid is not None:
                fstat = os.stat("/proc/%s"%procNum)
                if fstat.st_uid != uid:
                    return ""
            if path.exists(cmdLineFile):
                if not_chroot:
                    root_link = '/proc/%s/root' % procNum
                    if os.readlink(root_link) != "/":
                        return ""
                if chroot is not None:
                    root_link = '/proc/%s/root' % procNum
                    if os.readlink(root_link) != chroot:
                        return ""
                return readFile(cmdLineFile).strip()
        except Exception:
            pass
        return ""

    if not os.access('/proc', os.R_OK):
        return []
    if withpid:
        return filter(lambda x: x[1],
                      map(lambda x:(x,getCmd(x)),
                          filter(lambda x: x.isdigit(),
                                 listDirectory('/proc'))))
    else:
        return filter(None,
                      map(getCmd,
                          filter(lambda x: x.isdigit(),
                                 listDirectory('/proc'))))