|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# Copyright 2008-2016 Mir Calculate. http://www.calculate-linux.org
|
|
|
#
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
# You may obtain a copy of the License at
|
|
|
#
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
#
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
# See the License for the specific language governing permissions and
|
|
|
# limitations under the License.
|
|
|
|
|
|
from calculate.lib.utils.files import (readLinesFile, readFile, pathJoin,
|
|
|
process, find)
|
|
|
from calculate.lib.utils.portage import (reVerSplit, EmergePackage,
|
|
|
getInstalledAtom)
|
|
|
from os import path
|
|
|
import os
|
|
|
import glob
|
|
|
import re
|
|
|
import hashlib
|
|
|
from itertools import groupby
|
|
|
from collections import defaultdict
|
|
|
|
|
|
|
|
|
class DbPkg(object):
|
|
|
db_path = 'var/db/pkg'
|
|
|
|
|
|
|
|
|
def fixpath(fn, prefix='/'):
|
|
|
if prefix == '/':
|
|
|
return fn
|
|
|
return fn[len(prefix):]
|
|
|
|
|
|
|
|
|
class ContentsError(Exception):
|
|
|
pass
|
|
|
|
|
|
|
|
|
class ContentsFormat(object):
|
|
|
"""
|
|
|
Функция для конвертации данных из CONTENT в словарь и обратно
|
|
|
"""
|
|
|
prefix = "/"
|
|
|
|
|
|
reObj = re.compile(
|
|
|
r"^(?:"
|
|
|
r"(?:(?P<sym>sym)\s+(?P<symname>.*)\s+->"
|
|
|
r"\s+(?P<symtarget>.*)\s+(?P<symtime>\S+)\s*)|"
|
|
|
r"(?:(?P<dirfif>dir|fif)\s+(?P<dirname>.*)\s*)|"
|
|
|
r"(?:(?P<obj>obj)\s+(?P<filename>.*)\s+"
|
|
|
r"(?P<md5>\S+)\s+(?P<filetime>\S+)\s*))$")
|
|
|
|
|
|
def _identifyLine(self, line):
|
|
|
res = self.reObj.search(line.strip())
|
|
|
if res:
|
|
|
res = res.groupdict()
|
|
|
if res.get('dirfif', ''):
|
|
|
return res['dirname'], {'type': res['dirfif']}
|
|
|
elif res.get('obj', ''):
|
|
|
return (res['filename'], {'type': "obj",
|
|
|
'md5': res['md5'],
|
|
|
'mtime': res['filetime']})
|
|
|
elif res.get('sym', ''):
|
|
|
return (res['symname'], {'type': 'sym',
|
|
|
'target': res['symtarget'],
|
|
|
'mtime': res['symtime']})
|
|
|
|
|
|
def _write_info(self, f, filename, value):
|
|
|
formats = {'obj': "obj {filename} {md5} {mtime}\n",
|
|
|
'sym': "sym {filename} -> {target} {mtime}\n",
|
|
|
'dir': "dir {filename}\n",
|
|
|
'fif': "fif {filename}\n"}
|
|
|
f.write(formats[value['type']].format(filename=filename, **value))
|
|
|
|
|
|
def _fixNameByPrefix(self, filename):
|
|
|
if self.prefix != '/' and filename.startswith(self.prefix):
|
|
|
return filename[len(self.prefix):]
|
|
|
return filename
|
|
|
|
|
|
|
|
|
class PkgContents(ContentsFormat):
|
|
|
"""
|
|
|
Object for work with CONTENTS file
|
|
|
"""
|
|
|
reCfg = re.compile(r"/\._cfg\d{4}_")
|
|
|
|
|
|
def __init__(self, pkg, prefix="/"):
|
|
|
self.prefix = prefix
|
|
|
self.contentFile = path.join(prefix, '%s/%s/CONTENTS' % (DbPkg.db_path,
|
|
|
pkg))
|
|
|
self.readContents()
|
|
|
|
|
|
def readContents(self):
|
|
|
"""
|
|
|
Re-read contents
|
|
|
"""
|
|
|
self.content = dict(filter(None, map(self._identifyLine,
|
|
|
readLinesFile(self.contentFile))))
|
|
|
|
|
|
def writeContents(self):
|
|
|
"""
|
|
|
Write current content
|
|
|
"""
|
|
|
f = open(self.contentFile, 'w')
|
|
|
for filename in sorted(
|
|
|
self.content.keys(),
|
|
|
key=lambda x: (1 if x.startswith('/etc') else 0, x)):
|
|
|
value = self.content[filename]
|
|
|
self._write_info(f, filename, value)
|
|
|
f.close()
|
|
|
|
|
|
def addDir(self, filename):
|
|
|
filename = self.reCfg.sub("/", filename)
|
|
|
if filename != '/':
|
|
|
if (filename not in self.content or
|
|
|
self.content[filename]['type'] != 'dir'):
|
|
|
self.addDir(path.dirname(filename))
|
|
|
self.content[filename] = {'type': 'dir'}
|
|
|
|
|
|
def addFile(self, filename):
|
|
|
newfilename = pathJoin(self.prefix, filename)
|
|
|
filename = self.reCfg.sub("/", filename)
|
|
|
self.content[filename] = {'type': 'obj',
|
|
|
'md5': hashlib.md5(
|
|
|
readFile(newfilename)).hexdigest(),
|
|
|
'mtime': str(
|
|
|
int(os.stat(newfilename).st_mtime))}
|
|
|
|
|
|
def addLink(self, filename):
|
|
|
newfilename = pathJoin(self.prefix, filename)
|
|
|
filename = self.reCfg.sub("/", filename)
|
|
|
self.content[filename] = {'type': 'sym',
|
|
|
'target': os.readlink(newfilename),
|
|
|
'mtime': str(
|
|
|
int(os.lstat(newfilename).st_mtime))}
|
|
|
|
|
|
def origFileName(self, filename):
|
|
|
return self.reCfg.sub("/", filename)
|
|
|
|
|
|
def removeObject(self, filename):
|
|
|
filename = self._fixNameByPrefix(filename)
|
|
|
filename = self.reCfg.sub("/", filename)
|
|
|
if filename in self.content:
|
|
|
self.content.pop(filename)
|
|
|
|
|
|
def addObject(self, filename):
|
|
|
"""
|
|
|
Add object to content
|
|
|
"""
|
|
|
if filename != '/':
|
|
|
filename = self._fixNameByPrefix(filename)
|
|
|
newfilename = pathJoin(self.prefix, filename)
|
|
|
self.addDir(path.dirname(filename))
|
|
|
if path.islink(newfilename):
|
|
|
self.addLink(filename)
|
|
|
elif path.isdir(newfilename):
|
|
|
self.addDir(filename)
|
|
|
elif path.isfile(newfilename):
|
|
|
self.addFile(filename)
|
|
|
|
|
|
|
|
|
def checkReserved(fileName, contentFile):
|
|
|
"""
|
|
|
Check contents with newContent
|
|
|
"""
|
|
|
TYPE, FILENAME, MD5, MTIME = 0, 1, 2, 3
|
|
|
obj = filter(lambda x: x[1] == fileName,
|
|
|
map(lambda x: x.split(' '),
|
|
|
filter(lambda x: x.startswith('obj'),
|
|
|
readLinesFile(contentFile))))
|
|
|
# if pkg not content filename
|
|
|
if not obj:
|
|
|
return True
|
|
|
# if file is not exists
|
|
|
if not path.exists(fileName):
|
|
|
return True
|
|
|
contentMD5 = hashlib.md5(readFile(fileName)).hexdigest().strip()
|
|
|
configMD5 = obj[0][MD5].strip()
|
|
|
# if content was not changed
|
|
|
if contentMD5 == configMD5:
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
|
|
|
def checkContents(pkg, fileName, prefix='/', reservedFile=None):
|
|
|
"""
|
|
|
Check contents with newContent
|
|
|
"""
|
|
|
contentFile = path.join(prefix,
|
|
|
'%s/%s/CONTENTS' % (DbPkg.db_path, pkg))
|
|
|
if prefix != '/' and fileName.startswith(prefix):
|
|
|
shortName = fileName[len(prefix):]
|
|
|
else:
|
|
|
shortName = fileName
|
|
|
|
|
|
TYPE, FILENAME, MD5, MTIME = 0, 1, 2, 3
|
|
|
obj = filter(lambda x: x[1] == shortName,
|
|
|
map(lambda x: x.split(' '),
|
|
|
filter(lambda x: x.startswith('obj'),
|
|
|
readLinesFile(contentFile))))
|
|
|
# if pkg not content filename
|
|
|
if not obj:
|
|
|
# for using reserved -CONTENTS file on postinst
|
|
|
if not reservedFile or checkReserved(fileName, reservedFile):
|
|
|
return True
|
|
|
else:
|
|
|
return False
|
|
|
# if file is not exists
|
|
|
if not path.exists(fileName):
|
|
|
# for using reserved -CONTENTS file on postinst
|
|
|
if not reservedFile or checkReserved(fileName, reservedFile):
|
|
|
return True
|
|
|
else:
|
|
|
return False
|
|
|
contentMD5 = hashlib.md5(readFile(fileName)).hexdigest().strip()
|
|
|
configMD5 = obj[0][MD5].strip()
|
|
|
# if content was not changed
|
|
|
if contentMD5 == configMD5:
|
|
|
return True
|
|
|
return False
|
|
|
|
|
|
|
|
|
def getCfgFiles(protected_dirs=('/etc',), prefix='/'):
|
|
|
"""
|
|
|
Get protected cfg files
|
|
|
"""
|
|
|
reCfg = re.compile(r"/\._cfg\d{4}_", re.S)
|
|
|
findParams = ["find"] + map(lambda x: pathJoin(prefix, x),
|
|
|
protected_dirs) + [
|
|
|
"-name", "._cfg????_*", "!", "-name", ".*~", "!", "-iname",
|
|
|
".*.bak", "-printf", r"%T@ %p\n"]
|
|
|
mapCfg = {}
|
|
|
for filetime, sep, filename in map(lambda x: x.partition(' '),
|
|
|
filter(None, process(*findParams))):
|
|
|
origFilename = reCfg.sub(r'/', filename)
|
|
|
if not origFilename in mapCfg:
|
|
|
mapCfg[origFilename] = []
|
|
|
mapCfg[origFilename].append((int(filetime.split('.')[0]), filename))
|
|
|
return mapCfg
|
|
|
|
|
|
|
|
|
def fillContents(allContent, protected, prefix='/'):
|
|
|
"""
|
|
|
Fill dict file - package
|
|
|
"""
|
|
|
dbPath = pathJoin(prefix, DbPkg.db_path)
|
|
|
for contentFile in glob.glob(dbPath + "/*/*/CONTENTS"):
|
|
|
for objFile in filter(lambda x: x.startswith('obj '),
|
|
|
readLinesFile(contentFile)):
|
|
|
res = PkgContents.reObj.search(objFile.strip())
|
|
|
if res:
|
|
|
fn = res.groupdict()['filename']
|
|
|
if filter(lambda x: fn.startswith(x), protected):
|
|
|
pkg = reVerSplit.search(os.path.dirname(contentFile))
|
|
|
if pkg:
|
|
|
pkg = "%s/%s" % (pkg.groups()[:2])
|
|
|
allContent[fn] = pkg
|
|
|
|
|
|
|
|
|
class FileOwners(object):
|
|
|
"""
|
|
|
Объект позволяющий определить какому пакету принадлежит указанный файл
|
|
|
"""
|
|
|
|
|
|
def __init__(self, prefix='/'):
|
|
|
self.prefix = prefix
|
|
|
self.db_path = path.join(prefix, DbPkg.db_path)
|
|
|
self.file_owners = defaultdict(list)
|
|
|
self.initialize()
|
|
|
|
|
|
def content_files(self):
|
|
|
for x in glob.glob(path.join(self.db_path, '*/*/CONTENTS')):
|
|
|
yield x
|
|
|
|
|
|
def initialize(self):
|
|
|
for content_file in self.content_files():
|
|
|
#print "DEBUG:", content_file
|
|
|
slot_file = path.join(path.dirname(content_file), "SLOT")
|
|
|
slot = readFile(slot_file).strip()
|
|
|
cpv, ebuild = path.split(slot_file)
|
|
|
cpv = cpv[len(self.db_path) + 1:]
|
|
|
content_data = PkgContents(cpv, self.prefix).content
|
|
|
for k in content_data:
|
|
|
content_data[k]['package'] = EmergePackage(
|
|
|
"%s:%s" % (cpv, slot))
|
|
|
self.file_owners[k].append(content_data[k])
|
|
|
|
|
|
def get_owners(self, fn, prefix='/'):
|
|
|
"""
|
|
|
Получить пакет к которому принадлежит указанный файл
|
|
|
:return:
|
|
|
"""
|
|
|
info = self.get_info(fn, prefix)
|
|
|
if info:
|
|
|
return [x['package'] for x in info]
|
|
|
return []
|
|
|
|
|
|
def get_info(self, fn, prefix="/"):
|
|
|
"""
|
|
|
Получить данные (словарь) 'md5', тип, пакет
|
|
|
:param fn:
|
|
|
:param prefix:
|
|
|
:return:
|
|
|
"""
|
|
|
fn = fixpath(fn, prefix)
|
|
|
return self.file_owners.get(fn, [])
|
|
|
|
|
|
|
|
|
class FileOwnersRestricted(FileOwners):
|
|
|
"""
|
|
|
Объект считывающий не все CONTENTS из базы, а только те, которые содержат
|
|
|
любой объект из списка файлов
|
|
|
"""
|
|
|
|
|
|
def __init__(self, prefix='/', files=()):
|
|
|
self.files = list(files)
|
|
|
super(FileOwnersRestricted, self).__init__(prefix)
|
|
|
|
|
|
def content_files(self):
|
|
|
if self.files:
|
|
|
searcher = re.compile("|".join(" %s " % x for x in self.files))
|
|
|
for fn in glob.glob(path.join(self.db_path, '*/*/CONTENTS')):
|
|
|
if searcher.search(readFile(fn)):
|
|
|
yield fn
|
|
|
|
|
|
class ContentsStorage(ContentsFormat):
|
|
|
"""
|
|
|
Объект сохранения CONTENT информации по отдельным файлам. Структура файла
|
|
|
похожа на CONTENTS только первой колонкой добавлено название пакета со
|
|
|
слотом
|
|
|
"""
|
|
|
|
|
|
def __init__(self, storage_file, owner=None):
|
|
|
self.storage_file = storage_file
|
|
|
self.owner = owner
|
|
|
|
|
|
def keep(self, dn, prefix='/'):
|
|
|
"""
|
|
|
Просканировать указанный dn и сохранить информация в хранилище
|
|
|
:param dn:
|
|
|
:param prefix:
|
|
|
:return:
|
|
|
"""
|
|
|
if self.owner is None:
|
|
|
raise ContentsError("Owner not prepared for contents keeping")
|
|
|
with open(self.storage_file, 'w') as f:
|
|
|
for fn in find(dn):
|
|
|
info = self.owner.get_info(fn, prefix)
|
|
|
if not info:
|
|
|
continue
|
|
|
if info[0]['type'] == 'dir':
|
|
|
continue
|
|
|
self._write_entry(f, fixpath(fn, prefix), info)
|
|
|
|
|
|
def _write_entry(self, f, fn, info):
|
|
|
for data in info:
|
|
|
f.write("{CATEGORY}/{PN}:{SLOTONLY} ".format(**data['package']))
|
|
|
self._write_info(f, fn, data)
|
|
|
|
|
|
def _get_packages(self, prefix='/'):
|
|
|
for package, objinfos in groupby(
|
|
|
sorted(readLinesFile(self.storage_file)),
|
|
|
lambda x: x.partition(" ")[0]):
|
|
|
package = list(getInstalledAtom(package))
|
|
|
if not package:
|
|
|
list(getInstalledAtom(package[0].partition(":")[0]))
|
|
|
if package:
|
|
|
package = package[-1]
|
|
|
content = PkgContents(str(package), prefix)
|
|
|
yield content, objinfos
|
|
|
|
|
|
def restore(self, prefix='/'):
|
|
|
"""
|
|
|
Восстановить записи о файлах принадлежащих пакету.
|
|
|
:param prefix:
|
|
|
:return:
|
|
|
"""
|
|
|
for content, objinfos in self._get_packages(prefix):
|
|
|
for infoline in (x.strip().partition(' ')[2] for x in objinfos):
|
|
|
fn, data = self._identifyLine(infoline)
|
|
|
if path.exists(fn):
|
|
|
content.content[fn] = data
|
|
|
content.addDir(path.dirname(fn))
|
|
|
os.utime(fn, (int(data['mtime']), int(data['mtime'])))
|
|
|
content.writeContents()
|
