From e1b03589448bafdc10fe21ac84573a750529422a Mon Sep 17 00:00:00 2001 From: Mike Hiretsky Date: Fri, 1 Feb 2013 16:46:40 +0400 Subject: [PATCH] _cfg compatibility --- calculate/lib/cl_template.py | 149 ++++++++++++++++++++++++++++++--- calculate/lib/utils/content.py | 39 ++++++++- calculate/lib/utils/portage.py | 15 ++-- calculate/lib/variables/env.py | 26 ++++++ 4 files changed, 211 insertions(+), 18 deletions(-) diff --git a/calculate/lib/cl_template.py b/calculate/lib/cl_template.py index f540962..2ca8405 100644 --- a/calculate/lib/cl_template.py +++ b/calculate/lib/cl_template.py @@ -30,6 +30,7 @@ import random import string import time import glob +import hashlib import fcntl from itertools import * # < <= == != >= > @@ -38,7 +39,7 @@ from operator import lt, le, eq, ne, ge, gt from utils.common import _error, _warning from utils.text import _toUNICODE, convertStrListDict from utils.portage import isPkgInstalled,reVerSplitToPV -from utils.content import PkgContents +from utils.content import PkgContents,checkContents,getCfgFiles from utils.files import (getModeFile, listDirectory,removeDir, typeFile, scanDirectory, pathJoin,readFile,readLinesFile,process,STDOUT) @@ -2232,7 +2233,12 @@ class _file(_error): self.F_CONF = self.__openConfFile(self.nameFileConfig) if self.F_TEMPL and self.F_CONF: self.textTemplate = self.F_TEMPL.read() - self.textConfig = self.F_CONF.read() + if self.configMode == T_NEWCFG: + origConfigName = re.sub(r'/._cfg\d{4}_([^/]+)$','/\\1', + self.nameFileConfig) + self.textConfig = readFile(origConfigName) + else: + self.textConfig = self.F_CONF.read() def __del__(self): self.closeFiles() @@ -3775,6 +3781,12 @@ class ChangedFiles: filter(lambda x:x[1], map(lambda x:(x[0],filter(lambda x:x[0] == pkg,x[1])),self.data.items()))) +# modes work with configuration file +# T_ORIGIN - work with original config file +# T_CFG - work with last ._cfg file +# T_NEWCFG - new content has difference with (create new ._cfg file) +T_ORIGIN, T_CFG, T_NEWCFG = 0,1,2 + class Template(_file,_terms,_warning,xmlShare,templateFormat,_shareTemplate): """Класс для работы с шаблонами @@ -3794,7 +3806,10 @@ class Template(_file,_terms,_warning,xmlShare,templateFormat,_shareTemplate): def __init__(self, objVar, servDir=False, dirsFilter=[], filesFilter=[], cltObj=True, cltFilter=True, printWarning=True, printSUCCESS=lambda x:x,printWARNING=lambda x:x, - printERROR=lambda x:x,askConfirm=lambda x:x): + printERROR=lambda x:x,askConfirm=lambda x:x, + userProfile=False,dispatchConf=None): + self.userProfile = userProfile + self.dispatchConf = dispatchConf self.changedFiles = ChangedFiles() self.printSUCCESS = printSUCCESS self.printERROR = printERROR @@ -4196,6 +4211,7 @@ gettext -d cl_template "$*" origstat = os.stat(origfilename)[stat.ST_CTIME] newstat = os.stat(filename)[stat.ST_CTIME] if newstat > origstat: + self.configMode = T_CFG return filename return origfilename return origfilename @@ -4263,8 +4279,8 @@ gettext -d cl_template "$*" if skipDirs or skipTemplates: # print warning from cl_print import color_print - self.printWARNING(_("No conditions for checking the value of variable" - " 'cl_name'")) + self.printWARNING(_("No conditions for checking the value of " + "an action variable")) skipDirTemplates = [] for skipDir in skipDirs: skipTempl = os.path.join(skipDir,self.templDirNameFile) @@ -4281,9 +4297,9 @@ gettext -d cl_template "$*" setWARNING("") setWARNING(_("Headers of directory templates and headers " "of files on the first level should include " - "the 'cl_name' variable or an action variable.")) + "an action variable.")) setWARNING(_("Example:")) - setWARNING("# Calculate cl_name==calculate-install") + setWARNING("# Calculate ac_install_merge==on") return skipDirs + skipTemplates def applyTemplates(self,progress=True,rerun=True): @@ -4449,8 +4465,76 @@ gettext -d cl_template "$*" nameTemplate) return False self.queueExecute = [] + if not self.userProfile: + self.updateProtectedFiles() return (self.createdDirs, self.filesApply) + def updateProtectedFiles(self): + """ + Update ._cfg0000 files + """ + chrootPath = self.objVar.Get('cl_chroot_path') + cfgs = getCfgFiles(self.objVar.Get('cl_config_protect'), + prefix=chrootPath) + autoUpdateDict = {} + for pkg in list(set(self.changedFiles.getPkgs())): + category = isPkgInstalled(pkg,prefix=chrootPath) + if category: + pkgContents = PkgContents("{CATEGORY}/{PF}".format( + **category[0]),prefix=chrootPath) + for filename,action in self.changedFiles.getPkgFiles(pkg): + if filename in self.protectedFiles: + pkgContents.removeObject(filename) + continue + if action in (ChangedFiles.FILE_MODIFIED, + ChangedFiles.DIR_CREATED, + ChangedFiles.DIR_EXISTS): + pkgContents.addObject(filename) + elif action in (ChangedFiles.FILE_REMOVED, + ChangedFiles.DIR_REMOVED): + pkgContents.removeObject(filename) + files = set(map(lambda x:pathJoin(chrootPath,x), + pkgContents.content.keys())) + if self.objVar.Get('cl_autoupdate_set') == 'off': + notUpdate = files - set(self.autoUpdateFiles) + files &= set(self.autoUpdateFiles) + for filename in list(notUpdate&set(cfgs.keys())): + if hashlib.md5(readFile(filename)).hexdigest() == \ + hashlib.md5(readFile( + cfgs[filename][0][1])).hexdigest(): + files.add(filename) + print "Added ",filename + + for filename in list(files&set(cfgs.keys())): + # get ctime from orig filename + if os.path.exists(filename): + ctime = os.stat(filename).st_ctime + else: + ctime = 0 + # if orig filename older that .cfg + cfgs[filename].sort(reverse=True) + if ctime < cfgs[filename][0][0]: + try: + open(filename,'w').write( + readFile(cfgs[filename][0][1])) + except Exception as e: + self.printERROR(str(e)) + self.printWARNING( + _("Failed to copy {ffrom} to {fto}").format( + ffrom=cfgs[filename][0][1],fto=filename)) + continue + autoUpdateDict[cfgs[filename][0][1]] = filename + for mtime,fn in cfgs[filename]: + try: + os.unlink(fn) + except Exception as e: + self.printWARNING(_("Failed to remove %s")%fn) + pkgContents.writeContents() + self.filesApply = map(lambda x:autoUpdateDict.get(x,x),self.filesApply) + if filter(lambda x:"._cfg" in x, self.filesApply): + self.printWARNING(_("Some config files need updating. Perform dispatch-conf.")) + if self.dispatchConf: + self.dispatchConf(self.filesApply) def scanningTemplates(self, scanDir, prefix=None, flagDir=False, optDir={}, skipTemplates=[]): @@ -4882,6 +4966,41 @@ gettext -d cl_template "$*" return int(mapGid[strGid]) return None + def checkOnNewConfigName(self,pathFile): + """ + Check on need update and return pathFile + """ + # if file in PROTECT_MASK or not in PROTECT + chrootPath = self.objVar.Get('cl_chroot_path') + if not filter(pathFile.startswith, + map(lambda x:pathJoin(chrootPath,x), + self.objVar.Get('cl_config_protect'))) or \ + filter(pathFile.startswith, + map(lambda x:pathJoin(chrootPath,x), + self.objVar.Get('cl_config_protect_mask'))): + return pathFile + # if file was already modified by templates + if pathFile in self.changedFiles.data.keys(): + return pathFile + # if using already created ._cfg file + if self.configMode != T_ORIGIN: + return pathFile + # not current package file + pkg = self.functObj.currentBelong + if not pkg: + return pathFile + pkg = isPkgInstalled(pkg,sortByVersion=True,prefix=chrootPath) + if not pkg: + return pathFile + if checkContents("{CATEGORY}/{PF}".format(**pkg[-1]), + pathFile, + prefix=chrootPath): + return pathFile + real_filename = os.path.basename(pathFile) + real_dirname = os.path.dirname(pathFile) + self.configMode = T_NEWCFG + return os.path.join(real_dirname,"._cfg0000_%s"%real_filename) + def getApplyHeadTemplate(self, nameFileTemplate, nameFileConfig, templateFileType, optFile): """Применяет заголовок к шаблону (права, владелец, и.т. д)""" @@ -5011,7 +5130,10 @@ gettext -d cl_template "$*" pathOldFile = pathJoin(path,nameFile) else: pathOldFile = pathJoin(path,os.path.split(nameFileConfig)[1]) - pathOldFile = self.fixNameFileConfig(pathOldFile) + pathOrigFile = pathOldFile + if not self.userProfile: + pathOldFile = self.fixNameFileConfig(pathOldFile) + pathOldFile = self.checkOnNewConfigName(pathOldFile) applyFiles = [pathOldFile] # Фильтрация шаблонов по названию файла realPath = os.path.join("/",pathOldFile.partition(self._baseDir)[2]) @@ -5037,7 +5159,7 @@ gettext -d cl_template "$*" ": " +\ nameFileTemplate) return ([], False) - + # Очищаем оригинальный файл if typeAppendTemplate == "clear": try: @@ -5050,27 +5172,31 @@ gettext -d cl_template "$*" return (applyFiles, False) # Удаляем оригинальный файл if typeAppendTemplate == "remove": + if objHeadNew.params.has_key("force"): + pathOldFile = pathOrigFile if os.path.islink(pathOldFile): # удаляем ссылку try: os.unlink(pathOldFile) + return (applyFiles, False) except: self.setError(_("Template error") + ": " +\ nameFileTemplate) self.setError(_("Failed to delete the link") + ": " +\ pathOldFile) return ([], False) - if os.path.isfile(pathOldFile): + if os.path.isfile(pathOldFile) and self.configMode == T_ORIGIN: # удаляем файл try: os.remove(pathOldFile) + return (applyFiles, False) except: self.setError(_("Template error") + ": " +\ nameFileTemplate) self.setError(_("Failed to delete the file") + ": " +\ pathOldFile) return ([], False) - return (applyFiles, False) + return ([], False) # Пропускаем обработку шаблона elif typeAppendTemplate == "skip": return ([], False) @@ -5370,6 +5496,7 @@ gettext -d cl_template "$*" self.nameFileTemplate = os.path.abspath(nameFileTemplate) self.F_TEMPL = self.openTemplFile(self.nameFileTemplate) self.textTemplate = self.F_TEMPL.read() + self.configMode = T_ORIGIN self.closeTemplFile() # Флаг копирования шаблона в конфигурационный файл flagCopyTemplate = True diff --git a/calculate/lib/utils/content.py b/calculate/lib/utils/content.py index 5d15676..c8d316a 100644 --- a/calculate/lib/utils/content.py +++ b/calculate/lib/utils/content.py @@ -98,7 +98,13 @@ class PkgContents: 'target':os.readlink(newfilename), 'mtime':str(int(os.lstat(newfilename).st_mtime))} + def _fixNameByPrefix(self,filename): + if self.prefix != '/' and filename.startswith(self.prefix): + return filename[len(self.prefix):] + return filename + def removeObject(self,filename): + filename = self._fixNameByPrefix(filename) filename = self.reCfg.sub("/",filename) if filename in self.content: self.content.pop(filename) @@ -107,6 +113,7 @@ class PkgContents: """ Add object to content """ + filename = self._fixNameByPrefix(filename) newfilename = pathJoin(self.prefix,filename) if filename != '/': self.addDir(path.dirname(filename)) @@ -117,12 +124,40 @@ class PkgContents: elif path.isfile(newfilename): self.addFile(filename) -def getCfgFiles(protected_dirs=['/etc']): +def checkContents(pkg,fileName,prefix='/'): + """ + Check contents with newContent + """ + contentFile = path.join(prefix,'var/db/pkg/%s/CONTENTS'%pkg) + if prefix != '/' and fileName.startswith(prefix): + shortName = fileName[len(prefix):] + else: + shortName = fileName + + TYPE,FILENAME,MD5,MTIME=0,1,2,3 + obj = filter(lambda x:x[1] == shortName, + map(lambda x:x.split(' '), + filter(lambda x:x.startswith('obj'), + readLinesFile(contentFile)))) + # if pkg not content filename + if not obj: + return True + # if file is not exists + if not path.exists(fileName): + return True + contentMD5 = hashlib.md5(readFile(fileName)).hexdigest().strip() + configMD5 = obj[0][MD5].strip() + # if content was not changed + if contentMD5 == configMD5: + return True + return False + +def getCfgFiles(protected_dirs=['/etc'],prefix='/'): """ Get protected cfg files """ reCfg = re.compile(r"/\._cfg\d{4}_",re.S) - findParams = ["find"]+protected_dirs+\ + findParams = ["find"]+map(lambda x:pathJoin(prefix,x),protected_dirs)+\ ["-name","._cfg????_*","!","-name",".*~","!","-iname",".*.bak", "-printf",r"%T@ %p\n"] mapCfg = {} diff --git a/calculate/lib/utils/portage.py b/calculate/lib/utils/portage.py index 5f0c359..215aea7 100644 --- a/calculate/lib/utils/portage.py +++ b/calculate/lib/utils/portage.py @@ -21,6 +21,7 @@ import sys import re from os import path from files import listDirectory, readFile +from common import getTupleVersion from calculate.lib.cl_lang import setLocalTranslate setLocalTranslate('cl_lib3',sys.modules[__name__]) @@ -69,15 +70,19 @@ def getPkgUses(fullpkg): filter(lambda x:x, iuse))) -def isPkgInstalled(pkg,prefix='/'): +def isPkgInstalled(pkg,prefix='/',sortByVersion=False): """Check is package installed""" pkgDir = path.join(prefix,'var/db/pkg') if "/" in pkg: category,op,pkg = pkg.partition('/') - return map(lambda x:x.update({'CATEGORY':category}) or x, - filter(lambda x:x['PN'] == pkg, - map(reVerSplitToPV, - listDirectory(path.join(pkgDir,category))))) + res = map(lambda x:x.update({'CATEGORY':category}) or x, + filter(lambda x:x['PN'] == pkg, + map(reVerSplitToPV, + listDirectory(path.join(pkgDir,category))))) + if len(res)>1 and sortByVersion: + return sorted(res,key=lambda x:getTupleVersion(x['PVR'])) + else: + return res else: return filter(lambda x: filter(lambda y:y['PN'] == pkg, map(reVerSplitToPV, diff --git a/calculate/lib/variables/env.py b/calculate/lib/variables/env.py index 76a844d..dd106ff 100644 --- a/calculate/lib/variables/env.py +++ b/calculate/lib/variables/env.py @@ -189,6 +189,14 @@ class VariableClAutoupdateSet(Variable): type = 'bool' value = 'off' + opt = ["--replace-conf"] + value = "off" + + def init(self): + self.help = _("replace protected configs") + self.label = _("Replace protected configs") + + class VariableClWsdl(Variable): """ Packages with wsdl @@ -216,6 +224,23 @@ class VariableClWsdlAvailable(ReadonlyVariable): retList.append("calculate-%s"%module) return retList +class VariableClConfigProtectMask(ReadonlyVariable): + """ + Value of CONFIG_PROTECT after source /etc/profile, and /etc append + """ + type = "list" + + def get(self): + displayEnv = process('/bin/bash',"-c","source /etc/profile;env", + stdout=PIPE) + for line in displayEnv: + if line.startswith("CONFIG_PROTECT_MASK="): + configProtectMask=line.rstrip().partition('=')[2].split() + break + else: + configProtectMask = [] + return configProtectMask + class VariableClConfigProtect(ReadonlyVariable): """ Value of CONFIG_PROTECT after source /etc/profile, and /etc append @@ -234,6 +259,7 @@ class VariableClConfigProtect(ReadonlyVariable): configProtect.append('/etc') return configProtect + class VariableClVerboseSet(Variable): """ Verbose output variable