calculate-utils-3-lib/pym/calculate/lib/utils/content.py

# -*- coding: utf-8 -*-

# Copyright 2008-2016 Mir Calculate. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from .files import (readLinesFile, readFile, pathJoin, process, find)
from .portage import (reVerSplit, EmergePackage, getInstalledAtom)
from os import path
import os
import glob
import re
import hashlib
import fnmatch
from itertools import groupby
from collections import defaultdict


class DbPkg(object):
    db_path = 'var/db/pkg'


def fixpath(fn, prefix='/'):
    if prefix == '/':
        return fn
    return fn[len(prefix):]


class ContentsError(Exception):
    pass


class ContentsFormat(object):
    """
    Функция для конвертации данных из CONTENT в словарь и обратно
    """
    prefix = "/"

    reObj = re.compile(
        r"^(?:"
        r"(?:(?P<sym>sym)\s+(?P<symname>.*)\s+->"
        r"\s+(?P<symtarget>.*)\s+(?P<symtime>\S+)\s*)|"
        r"(?:(?P<dirfif>dir|fif)\s+(?P<dirname>.*)\s*)|"
        r"(?:(?P<obj>obj)\s+(?P<filename>.*)\s+"
        r"(?P<md5>\S+)\s+(?P<filetime>\S+)\s*))$")

    def _identifyLine(self, line):
        res = self.reObj.search(line.strip())
        if res:
            res = res.groupdict()
            if res.get('dirfif', ''):
                return res['dirname'], {'type': res['dirfif']}
            elif res.get('obj', ''):
                return (res['filename'], {'type': "obj",
                                          'md5': res['md5'],
                                          'mtime': res['filetime']})
            elif res.get('sym', ''):
                return (res['symname'], {'type': 'sym',
                                         'target': res['symtarget'],
                                         'mtime': res['symtime']})

    def _write_info(self, f, filename, value):
        formats = {'obj': "obj {filename} {md5} {mtime}\n",
                   'sym': "sym {filename} -> {target} {mtime}\n",
                   'dir': "dir {filename}\n",
                   'fif': "fif {filename}\n"}
        f.write(formats[value['type']].format(filename=filename, **value))

    def _fixNameByPrefix(self, filename):
        if self.prefix != '/' and filename.startswith(self.prefix):
            return filename[len(self.prefix):]
        return filename


class PkgContents(ContentsFormat):
    """
    Object for work with CONTENTS file
    """
    reCfg = re.compile(r"/\._cfg\d{4}_")

    def __init__(self, pkg, db_path=DbPkg.db_path, prefix="/"):
        self.prefix = prefix
        self.contentFile = path.join(prefix, '%s/%s/CONTENTS' % (db_path, pkg))
        self.readContents()

    def readContents(self):
        """
        Re-read contents
        """
        self.content = dict(filter(None, map(self._identifyLine,
                                             readLinesFile(self.contentFile))))

    def writeContents(self):
        """
        Write current content
        """
        f = open(self.contentFile, 'w')
        for filename in sorted(
                self.content.keys(),
                key=lambda x: (1 if x.startswith('/etc') else 0, x)):
            value = self.content[filename]
            self._write_info(f, filename, value)
        f.close()

    def addDir(self, filename):
        filename = self.reCfg.sub("/", filename)
        if filename != '/':
            if (filename not in self.content or
                        self.content[filename]['type'] != 'dir'):
                self.addDir(path.dirname(filename))
                self.content[filename] = {'type': 'dir'}

    def addFile(self, filename):
        newfilename = pathJoin(self.prefix, filename)
        filename = self.reCfg.sub("/", filename)
        self.content[filename] = {'type': 'obj',
                                  'md5': hashlib.md5(
                                      readFile(newfilename)).hexdigest(),
                                  'mtime': str(
                                      int(os.stat(newfilename).st_mtime))}

    def addLink(self, filename):
        newfilename = pathJoin(self.prefix, filename)
        filename = self.reCfg.sub("/", filename)
        self.content[filename] = {'type': 'sym',
                                  'target': os.readlink(newfilename),
                                  'mtime': str(
                                      int(os.lstat(newfilename).st_mtime))}

    def origFileName(self, filename):
        return self.reCfg.sub("/", filename)

    def removeObject(self, filename):
        filename = self._fixNameByPrefix(filename)
        filename = self.reCfg.sub("/", filename)
        if filename in self.content:
            return self.content.pop(filename)

    def removeGlobObject(self, globfilename):
        globfilename = self._fixNameByPrefix(globfilename)
        remove_list = fnmatch.filter(self.content.keys(), globfilename)
        return [(fn, self.removeObject(fn)) for fn in remove_list]

    def updateData(self, filename, data):
        filename = self._fixNameByPrefix(filename)
        self.content[filename] = data

    def addObject(self, filename):
        """
        Add object to content
        """
        if filename != '/':
            filename = self._fixNameByPrefix(filename)
            newfilename = pathJoin(self.prefix, filename)
            self.addDir(path.dirname(filename))
            if path.islink(newfilename):
                self.addLink(filename)
            elif path.isdir(newfilename):
                self.addDir(filename)
            elif path.isfile(newfilename):
                self.addFile(filename)

    def clearEmptyDirs(self):
        """
        Удалить пустые директорий
        """
        used_dirs = set()
        for fn in (x for x in self.content.keys()
                   if self.content[x]['type'] != "dir"):
            dn = path.dirname(fn)
            while dn != '/':
                used_dirs.add(dn)
                dn = path.dirname(dn)
        remove_dirs = [x for x in self.content.keys()
                       if (self.content[x]['type'] == 'dir' and
                           x not in used_dirs)]
        for dn in remove_dirs:
            self.content.pop(dn)

def checkReserved(fileName, contentFile):
    """
    Check contents with newContent
    """
    TYPE, FILENAME, MD5, MTIME = 0, 1, 2, 3
    obj = filter(lambda x: x[1] == fileName,
                 map(lambda x: x.split(' '),
                     filter(lambda x: x.startswith('obj'),
                            readLinesFile(contentFile))))
    # if pkg not content filename
    if not obj:
        return True
    # if file is not exists
    if not path.exists(fileName):
        return True
    contentMD5 = hashlib.md5(readFile(fileName)).hexdigest().strip()
    configMD5 = obj[0][MD5].strip()
    # if content was not changed
    if contentMD5 == configMD5:
        return True
    return False


def checkContents(pkg, fileName, prefix='/', reservedFile=None):
    """
    Check contents with newContent
    """
    contentFile = path.join(prefix,
                            '%s/%s/CONTENTS' % (DbPkg.db_path, pkg))
    if prefix != '/' and fileName.startswith(prefix):
        shortName = fileName[len(prefix):]
    else:
        shortName = fileName

    TYPE, FILENAME, MD5, MTIME = 0, 1, 2, 3
    obj = filter(lambda x: x[1] == shortName,
                 map(lambda x: x.split(' '),
                     filter(lambda x: x.startswith('obj'),
                            readLinesFile(contentFile))))
    # if pkg not content filename
    if not obj:
        # for using reserved -CONTENTS file on postinst
        if not reservedFile or checkReserved(fileName, reservedFile):
            return True
        else:
            return False
    # if file is not exists
    if not path.exists(fileName):
        # for using reserved -CONTENTS file on postinst
        if not reservedFile or checkReserved(fileName, reservedFile):
            return True
        else:
            return False
    contentMD5 = hashlib.md5(readFile(fileName)).hexdigest().strip()
    configMD5 = obj[0][MD5].strip()
    # if content was not changed
    if contentMD5 == configMD5:
        return True
    return False


def getCfgFiles(protected_dirs=('/etc',), prefix='/'):
    """
    Get protected cfg files
    """
    reCfg = re.compile(r"/\._cfg\d{4}_", re.S)
    findParams = ["find"] + map(lambda x: pathJoin(prefix, x),
                                protected_dirs) + [
                     "-name", "._cfg????_*", "!", "-name", ".*~", "!", "-iname",
                     ".*.bak", "-printf", r"%T@ %p\n"]
    mapCfg = {}
    for filetime, sep, filename in map(lambda x: x.partition(' '),
                                       filter(None, process(*findParams))):
        origFilename = reCfg.sub(r'/', filename)
        if not origFilename in mapCfg:
            mapCfg[origFilename] = []
        mapCfg[origFilename].append((int(filetime.split('.')[0]), filename))
    return mapCfg


def fillContents(allContent, protected, prefix='/'):
    """
    Fill dict file - package
    """
    dbPath = pathJoin(prefix, DbPkg.db_path)
    for contentFile in glob.glob(dbPath + "/*/*/CONTENTS"):
        for objFile in filter(lambda x: x.startswith('obj '),
                              readLinesFile(contentFile)):
            res = PkgContents.reObj.search(objFile.strip())
            if res:
                fn = res.groupdict()['filename']
                if filter(lambda x: fn.startswith(x), protected):
                    pkg = reVerSplit.search(os.path.dirname(contentFile))
                    if pkg:
                        pkg = "%s/%s" % (pkg.groups()[:2])
                        allContent[fn] = pkg


class FileOwners(object):
    """
    Объект позволяющий определить какому пакету принадлежит указанный файл
    """

    def __init__(self, prefix='/'):
        self.prefix = prefix
        self.db_path = path.join(prefix, DbPkg.db_path)
        self.file_owners = defaultdict(list)
        self.initialize()

    def content_files(self):
        for x in glob.glob(path.join(self.db_path, '*/*/CONTENTS')):
            yield x

    def get_md5_failed(self, filter_f=None):
        if filter_f is None:
            filter_f = lambda x: True
        for fn in (x for x in self.file_owners if filter_f(x)):
            fn_info = self.file_owners[fn][0]
            if not path.islink(fn):
                if path.isfile(fn):
                    system_md5 = hashlib.md5(readFile(fn)).hexdigest().strip()
                    if system_md5 != fn_info.get('md5', ''):
                        yield fn
            else:
                if path.lexists(fn):
                    system_target = os.readlink(fn)
                    if system_target != fn_info.get('target', ''):
                        yield fn

    def initialize(self):
        for content_file in self.content_files():
            slot_file = path.join(path.dirname(content_file), "SLOT")
            slot = readFile(slot_file).strip()
            cpv, ebuild = path.split(slot_file)
            cpv = cpv[len(self.db_path) + 1:]
            content_data = PkgContents(cpv, prefix=self.prefix).content
            for k in content_data:
                content_data[k]['package'] = EmergePackage(
                    "%s:%s" % (cpv, slot))
                self.file_owners[k].append(content_data[k])

    def get_owners(self, fn, prefix='/'):
        """
        Получить пакет к которому принадлежит указанный файл
        :return:
        """
        info = self.get_info(fn, prefix)
        if info:
            return [x['package'] for x in info]
        return []

    def get_info(self, fn, prefix="/"):
        """
        Получить данные (словарь) 'md5', тип, пакет
        :param fn:
        :param prefix:
        :return:
        """
        fn = fixpath(fn, prefix)
        return self.file_owners.get(fn, [])


class FileOwnersRestricted(FileOwners):
    """
    Объект считывающий не все CONTENTS из базы, а только те, которые содержат
    любой объект из списка файлов
    """

    def __init__(self, prefix='/', files=()):
        self.files = list(files)
        super(FileOwnersRestricted, self).__init__(prefix)

    def content_files(self):
        if self.files:
            searcher = re.compile("|".join(" %s[\n ]" % x for x in self.files))
            for fn in glob.glob(path.join(self.db_path, '*/*/CONTENTS')):
                if searcher.search(readFile(fn)):
                    yield fn


class ContentsStorage(ContentsFormat):
    """
    Объект сохранения CONTENT информации по отдельным файлам. Структура файла
    похожа на CONTENTS только первой колонкой добавлено название пакета со
    слотом
    """

    def __init__(self, storage_file, owner=None):
        self.storage_file = storage_file
        self.owner = owner

    def keep(self, dn, prefix='/'):
        """
        Просканировать указанный dn и сохранить информация в хранилище
        :param dn:
        :param prefix:
        :return:
        """
        if self.owner is None:
            raise ContentsError("Owner not prepared for contents keeping")
        with open(self.storage_file, 'w') as f:
            for fn in find(dn):
                info = self.owner.get_info(fn, prefix)
                if not info:
                    continue
                if info[0]['type'] == 'dir':
                    continue
                self._write_entry(f, fixpath(fn, prefix), info)

    def _write_entry(self, f, fn, info):
        for data in info:
            f.write("{CATEGORY}/{PN}:{SLOTONLY} ".format(**data['package']))
            self._write_info(f, fn, data)

    def _get_packages(self, prefix='/'):
        for package, objinfos in groupby(
                sorted(readLinesFile(self.storage_file)),
                lambda x: x.partition(" ")[0]):
            pkgs = list(getInstalledAtom(package))
            if not pkgs:
                pkgs = list(getInstalledAtom(package[0].partition(":")[0]))
            if pkgs:
                pkg = pkgs[-1]
                content = PkgContents(str(pkg), prefix=prefix)
                yield content, objinfos

    def restore(self, prefix='/', files=None):
        """
        Восстановить записи о файлах принадлежащих пакету.
        :param prefix:
        :return:
        """
        if files is None:
            check = lambda x: True
        else:
            check = files.__contains__
        for content, objinfos in self._get_packages(prefix):
            for infoline in (x.strip().partition(' ')[2] for x in objinfos):
                fn, data = self._identifyLine(infoline)
                if path.exists(fn) and check(fn):
                    content.content[fn] = data
                    content.addDir(path.dirname(fn))
                    os.utime(fn, (int(data['mtime']), int(data['mtime'])))
            content.writeContents()