diff --git a/calculate/lib/cl_template.py b/calculate/lib/cl_template.py index d1fe4a4..0514188 100644 --- a/calculate/lib/cl_template.py +++ b/calculate/lib/cl_template.py @@ -18,7 +18,10 @@ import sys import os import stat import re -import xml.dom.minidom +import xml.dom.minidom as minidom +import xml +if hasattr(xml,"use_pyxml"): + xml.use_pyxml() from xml import xpath import subprocess import types @@ -1047,7 +1050,7 @@ class xmlDoc: docTxt += '%s'% version docTxt += '%s' % typeDoc docTxt += '' - self.doc = xml.dom.minidom.parseString(docTxt) + self.doc = minidom.parseString(docTxt) self.root = self.doc.documentElement self.body = xpath.Evaluate('child::body',self.root)[0] # установка разделителя областей diff --git a/calculate/lib/datavars.py b/calculate/lib/datavars.py index 7b36b43..92befae 100644 --- a/calculate/lib/datavars.py +++ b/calculate/lib/datavars.py @@ -10,6 +10,7 @@ from threading import RLock import pyinotify from itertools import * import operator +import traceback def itemgetter(*args,**kwargs): """ @@ -21,6 +22,14 @@ def itemgetter(*args,**kwargs): return lambda x:(x[args[0]],) return operator.itemgetter(*args) +def not_empty(itervalue): + """ + Check has iterator one or more element or not + """ + for i in itervalue: + return True + return False + class DataVarsError(Exception): """Exception of getting variable values""" @@ -91,17 +100,18 @@ class Variable: check_after = [] # fill value is true untrusted = False + NONE,CHECK,UNCOMPAT,HUMAN = range(4) def __init__(self,parent=None,**kwargs): """ Initialize instance """ - self.checking = False + self.modeGet = Variable.NONE self.invalid = self.value is None self.wasSet = False # variable is user or ini set self.reqVars = [] # varaibles which used this value for fill self.reqCheck = [] # variables which used this value for check - self.reqUncompat = [] # varaibles which used this value for uncompat + self.reqUncompat = [] # varaibles which used for uncompat self.variableType = type self.parent = parent self.name = self.getVariableName() @@ -111,7 +121,7 @@ class Variable: def init(self): """ - Overridible init + Overridible init using for palce translatable strings """ pass @@ -121,13 +131,29 @@ class Variable: def Get(self,varname=None,humanreadable=False): """Get value of other value""" - if not varname: - varname = self.name + if not varname or varname == self.name: + try: + oldmode = self.modeGet + self.modeGet = Variable.NONE + return self._get() + finally: + self.modeGet = oldmode + + varObj = self.parent.getInfo(varname) + #varObj.modeGet = Variable.UNCOMPAT res = self.parent.Get(varname,humanreadable) - if self.checking: - varObj = self.parent.getInfo(varname) - if not self in varObj.reqCheck: - varObj.reqCheck.append(self) + + # build dependence for check and uncompatible + if varObj: + if self.modeGet == Variable.NONE: + if not self in varObj.reqVars: + varObj.reqVars.append(self) + if self.modeGet == Variable.CHECK: + if not self in varObj.reqCheck: + varObj.reqCheck.append(self) + elif self.modeGet == Variable.UNCOMPAT: + if not varObj in self.reqUncompat: + self.reqUncompat.append(varObj) return res def Choice(self,varname): @@ -136,13 +162,13 @@ class Variable: def Select(self,selField,**kwargs): """Select by datavars""" - return self.parent.Select(selField,**kwargs) + return self.parent.Select(selField,zipVars=self.ZipVars,**kwargs) def ZipVars(self,*argvVarNames): """ Get zipped values of variables specified by list 'argvVarNames' """ - return self.parent.ZipVars(*argvVarNames) + return zip(*map(self.Get,argvVarNames)) def get(self): """ @@ -153,7 +179,23 @@ class Variable: def humanReadable(self): """Return human readable value""" - return self.Get(self.name) + return self.Get() + + def _human(self): + try: + self.modeGet = Variable.HUMAN + return self.humanReadable() + finally: + self.modeGet = Variable.NONE + + def getHumanReadableAuto(self): + """ + Return humanreadable values for readonly variables + """ + if self.mode == READONLY: + return self._human() + else: + return self._get() def _get(self): """ @@ -182,6 +224,16 @@ class Variable: """ return [] + def _choice(self): + """ + Convert choice created by variable choice to two tuple (values,comments) + """ + res = self.choice() + if res and type(res[0]) in (tuple,list): + return zip(*res) + else: + return (res,None) + def check(self,value): """ Overridable @@ -199,12 +251,25 @@ class Variable: """ return "" + def _uncompatible(self): + """ + Full check uncompatible + """ + try: + self.modeGet = Variable.UNCOMPAT + self.reqUncompat = [] + return self.uncompatible() + finally: + self.modeGet = Variable.NONE + def set(self,value): """ Overridable Using for replace value before set """ + if self.type == "bool": + value = "on" if self.isTrue(value) else "off" return value def checkType(self,value): @@ -217,10 +282,11 @@ class Variable: vartype="list")) error = _("Values for {varname} may be {vartype} only") else: - value = [value] + value = repeat(value,1) error = _("Value for {varname} may be {vartype} only") if "string" in self.type: - if filter(lambda x:not type(x) in (str,unicode), value): + value, valuecopy = tee(value,2) + for v in (x for x in valuecopy if not type(x) in (str,unicode)): raise VariableError(error.format( varname=self.label or self.name, vartype="string")) @@ -228,7 +294,7 @@ class Variable: if "choice" in self.type: choiceVal = self.choice() if choiceVal and type(choiceVal[0]) in (tuple,list): - choiceVal = map(lambda x:x[0],choiceVal) + choiceVal = [x[0] for x in choiceVal] for val in value: if "choice" in self.type and not "choiceedit" in self.type: if choiceVal and val and not val in choiceVal: @@ -237,39 +303,47 @@ class Variable: varname=name, vartype=formatListOr(choiceVal))) + def setValue(self,value): + """ + Standard action for set value + """ + self.value = self.set(value) + self.wasSet = True + self.invalid = False + # run check + self._check() + def _set(self,value,force=False): """ Standard inner method for setting value for variable. """ # runc check choice try: - self.checking = True + self.modeGet = Variable.CHECK self.checkType(value) finally: - self.checking = False + self.modeGet = Variable.NONE # if value change if value != self.value: self.invalidate(True) if hasattr(self.value,"close"): self.value.close() # run set value - self.value = self.set(value) - self.wasSet = True - self.invalid = False - # run check - self.checkValue() + self.setValue(value) - def checkValue(self): + def _check(self,value=None): """ Full check value """ + if value is None: + value = self.Get(self.name) try: - self.checking = True + self.modeGet = Variable.CHECK self.untrusted = True - self.check(self.value) + self.check(value) self.untrusted = False finally: - self.checking = False + self.modeGet = Variable.NONE def invalidate(self,force=False): """ @@ -289,16 +363,165 @@ class Variable: self.reqCheck = [] def isTrue(self,value): - if value.lower() in ('yes','on','true'): + if type(value) == bool: + return value + if value.lower() in ('yes','on','true'): return True return False +class TableVariable(Variable): + """ + Table type variable + """ + type = "table" + + def get(self,hr=False): + """Get table data""" + for varname,value in ifilter(lambda x:type(x[1]) != list, + imap(lambda x:(x,self.Get(x)), + self.source)): + raise VariableError( + _("Source variable %s not contains list")%varname) + return map(list, + izip_longest( + *map(lambda x:self.Get(x,humanreadable=hr), + self.source),fillvalue='')) or [[]] + + def getHumanReadableAuto(self): + """ + Return humanreadable values for readonly variables + """ + return self.get(hr=None) + + def humanReadable(self): + return self.get(hr=True) + + def __getWritableColumns(self,includeFirst=False): + """ + Get data for writable columns exclude index column + (Example: (1,'os_disk_mount) + """ + if includeFirst: + offset = 0 + else: + offset = 1 + return filter(lambda x:self.parent.getInfo(x[1]).mode == WRITEABLE, + enumerate(self.source[offset:])) + + def check(self,value): + """ + Check table value - check all writeable columns. + """ + writeCols = self.__getWritableColumns(includeFirst=True) + error = [] + if not any(value): + value = [[]]*len(writeCols) + else: + value = zip(*map(itemgetter(*map(itemgetter(0), + writeCols),alwaysTuple=True),value)) + for colInfo, values in \ + zip(writeCols,value): + try: + self.parent.Check(colInfo[1],values) + except VariableError as e: + error.append(e) + if error: + raise VariableError(error) + + def __isIndexWritable(self): + return self.parent.getInfo(self.source[0]).mode == WRITEABLE + + def checkType(self,value): + """Check table value type""" + # check type value (list and each element is list) + if not type(value) in (list,tuple) or \ + not_empty((i for i in value if not type(i) in (tuple,list))): + raise VariableError( + _("Value for {varname} may be '{vartype}' only").format( + varname=self.label or self.name, + vartype="table")) + # check len all entries + writeLen = len(self.__getWritableColumns())+1 + for item in value: + if item and len(item) != writeLen: + raise DataVarsError( + _("Wrong entry '{entry}' for table variable '{varname}'"). + format(entry=str(item), + varname=(self.label or self.name).lower())) + + # check rewrite readonly index + if not self.__isIndexWritable(): + indexValues = self.Get(self.source[0]) + for item in value: + if item and not item[0] in indexValues: + raise DataVarsError( + _("Attempt to rewrite a readonly index field " + "{fieldname} in variable {variablename}").format( + fieldname=self.source[0],variablename=self.name)) + + def setValue(self,value): + self.untrusted = True + oldvalue = self.Get(self.name) + # get writable columns + writeCols = self.__getWritableColumns() + # get slicer + _slice = itemgetter(*map(itemgetter(0),writeCols), + alwaysTuple=True) + # create dict for writable columns + # if table not empty + if any(oldvalue): + oldvalue = OrderedDict(map(lambda x:(x[0],_slice(x[1:])), + oldvalue)) + else: + oldvalue = OrderedDict() + # get new dict + if any(value): + if len(value[0]) == len(self.source): + newval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])), + value)) + else: + newval = OrderedDict(map(lambda x:(x[0],x[1:]),value)) + else: + newval = OrderedDict() + # if table with writable index field then replace all table + error = [] + if self.__isIndexWritable(): + if any(value): + self.parent.Check(self.source[0],zip(*value)[0]) + oldvalue = newval + try: + self.parent.Set(self.source[0],oldvalue.keys()) + except VariableError,e: + error.append(e) + # update entry by index field + else: + oldvalue.update(newval) + oldvalValues = zip(*oldvalue.values()) + for col,vals in zip(map(lambda x:x[1],writeCols), + oldvalValues): + try: + self.parent.Set(col,list(vals)) + except VariableError,e: + error.append(e) + for num,varname in writeCols[len(oldvalValues):]: + self.parent.Invalidate(varname,True) + if error: + raise VariableError(error) + else: + self.untrusted = False + class ReadonlyVariable(Variable): """ Alias for readonly variables """ mode = READONLY +class ReadonlyTableVariable(TableVariable): + """ + Alias for readonly table + """ + mode = READONLY + class FileVariable(Variable): """ Test variable @@ -344,6 +567,9 @@ class SimpleDataVars: self.cache[varname] = self.allVars.get(varname).get() return self.cache[varname] + def getInfo(self,varname): + return self.allVars.get(varname, None) + def flIniFileFrom(self,iniFile): """ Read variable values from ini files. @@ -555,30 +781,22 @@ class DataVars: varObj = self.loadVariables[varname] # if for this variable already use Get method if varObj.invalid and varObj in self.requestVariables: - varnames = "-".join(map(lambda x:x.name,self.requestVariables)) + varnames = "-".join(map(lambda x:x.name, + self.requestVariables+[varObj])) raise DataVarsError( _("Loop dependence of variables '%s'")%varnames) - # update dependence variable list for get variable - if self.requestVariables and not self.requestVariables[-1] in \ - varObj.reqVars: - varObj.reqVars.append(self.requestVariables[-1]) # add this variable for list of requested variables - if humanreadable is None and varObj.mode == 'r': - humanreadable = True if not humanreadable: self.requestVariables.append(varObj) - if varObj.source and Variable.get.__func__ == varObj.get.__func__: - retVal = self.__getTable(varObj,humanreadable) + if humanreadable is None: + res = varObj.getHumanReadableAuto() + elif humanreadable is True: + res = varObj._human() else: - if humanreadable: - retVal = varObj.humanReadable() - else: - retVal = varObj._get() - # after finish get value - pop variable from request list + res = varObj._get() if not humanreadable: self.requestVariables.pop() - #print "Variable(%s)=%s"%(varname,time.time()-tm) - return retVal + return res except BaseException,e: while(len(self.requestVariables)): self.requestVariables.pop() @@ -592,10 +810,7 @@ class DataVars: if not varname in self.loadVariables: self.loadVariable(varname) varObj = self.loadVariables[varname] - if varObj.source and Variable.get.__func__ == varObj.get.__func__: - self.__checkTable(varObj, value) - else: - varObj.checkValue() + varObj._check(value) def Choice(self, varname): """ @@ -605,7 +820,7 @@ class DataVars: if not varname in self.loadVariables: self.loadVariable(varname) varObj = self.loadVariables[varname] - return varObj.choice() + return varObj._choice()[0] def Uncompatible(self, varname): """ @@ -617,139 +832,32 @@ class DataVars: if not varname in self.loadVariables: self.loadVariable(varname) varObj = self.loadVariables[varname] - return varObj.uncompatible() - + return varObj._uncompatible() + + def getRequired(self,varname): + """ + Generate list variables using in uncompatible check + """ + added = [] + def genReqs(varObj): + if not varObj in added: + yield varObj + for var in varObj.reqVars: + for dep in genReqs(var): + yield dep + if hasattr(varObj,"source") and varObj.source: + for varsource in imap(self.getInfo,varObj.source): + for dep in genReqs(varsource): + yield dep + for dep in genReqs(self.getInfo(varname)): + added.append(dep) + return added def ChoiceAndComments(self,varname): - choiceVal = self.Choice(varname) - if choiceVal and type(choiceVal[0]) in (tuple,list): - return zip(*choiceVal) - else: - return (choiceVal,None) - - def __getTable(self,varobj,hr=False): - def setLen(lst): - if len(lst) < maxlen: - return lst+['']*(maxlen-len(lst)) - else: - return lst - maxlen = len(max(map(self.Get,varobj.source),key=len)) - wrongValues = filter(lambda x:type(x[1]) != list, - map(lambda x:(x,self.Get(x)), - varobj.source)) - if wrongValues: - raise DataVarsError( - _("Variable %s not contains list")%wrongValues[0][0]) - return map(lambda x:list(x), - zip(*map(setLen, - map(lambda x:self.Get(x,hr), - varobj.source)))) or [[]] - - def __checkTable(self,varObj,value): - """ - Check table value - check all writeable columns. - """ - self.__loadTableVariable(varObj) - writeCols = self.__getWritableColumns(varObj,includeFirst=True) - error = [] - if not any(value): - value = [[]]*len(writeCols) - for colInfo, values in \ - zip(writeCols, - zip(*map(itemgetter(*map(itemgetter(0), - writeCols),alwaysTuple=True),value))): - try: - self.Check(colInfo[1],values) - except VariableError as e: - error.append(e) - if error: - raise VariableError(error) - - def __loadTableVariable(self,varObj): - """ - Load all variables using in table - """ - map(self.loadVariable, - ifilterfalse(self.loadVariables.__contains__, - varObj.source)) - - def __getWritableColumns(self,varObj,includeFirst=False): - """ - Get data for writable columns exclude index column - (Example: (1,'os_disk_mount) - """ - if includeFirst: - offset = 0 - else: - offset = 1 - return filter(lambda x:self.loadVariables[x[1]].mode == WRITEABLE, - enumerate(varObj.source[offset:])) - - def __isIndexWritable(self,varObj): - return self.loadVariables[varObj.source[0]].mode == WRITEABLE - - def __setTable(self,varobj,varvalue): - """Set variable value to source table variable""" - # get current values - oldval = self.__Get(varobj.name) - # load column variables - self.__loadTableVariable(varobj) - # get writable columns - writeCols = self.__getWritableColumns(varobj) - # get slicer - _slice = itemgetter(*map(itemgetter(0),writeCols), - alwaysTuple=True) - # create dict for writable columns - # if table not empty - if any(oldval): - oldval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])), - oldval)) - else: - oldval = OrderedDict() - # get new dict - if any(varvalue): - if len(varvalue[0]) == len(varobj.source): - newval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])), - varvalue)) - else: - newval = OrderedDict(map(lambda x:(x[0],x[1:]), - varvalue)) - else: - newval = OrderedDict() - for item in newval.values(): - if len(item) > len(writeCols): - raise DataVarsError( - _("Wrong entry '{entry}' for table variable {varname}"). - format(entry=item,varname=varobj.name)) - # if table with writable index field then replace all table - error = [] - if self.__isIndexWritable(varobj): - if any(varvalue): - self.Check(varobj.source[0],zip(*varvalue)[0]) - oldval = newval - try: - self.Set(varobj.source[0],oldval.keys()) - except VariableError,e: - error.append(e) - # update entry by index field - else: - if not set(newval.keys()) <= set(oldval.keys()): - raise DataVarsError( - _("Attempt to rewrite a readonly index field " - "{fieldname} in variable {variablename}").format( - fieldname=varobj.source[0],variablename=varobj.name)) - oldval.update(newval) - oldvalValues = zip(*oldval.values()) - for col,vals in zip(map(lambda x:x[1],writeCols), - oldvalValues): - try: - self.Set(col,list(vals)) - except VariableError,e: - error.append(e) - for num,varname in writeCols[len(oldvalValues):]: - self.Invalidate(varname,True) - if error: - raise VariableError(error) + if not varname in self.loadVariables: + self.loadVariable(varname) + varObj = self.loadVariables[varname] + return varObj._choice() def getInfo(self, varname): """ @@ -763,19 +871,9 @@ class DataVars: """ Set value 'varvalue' for variable 'varname' """ - if not varname in self.loadVariables: - self.loadVariable(varname) - varObj = self.loadVariables[varname] + varObj = self.getInfo(varname) if force or varObj.mode != READONLY: - if varObj.source: - self.__setTable(varObj,varvalue) - # check on overriding method (variable has own fill) - if Variable.get.__func__ != varObj.get.__func__: - varObj._set(varvalue) - else: - if varObj.type == "bool" and type(varvalue) == bool: - varvalue = {True:'on',False:'off'}.get(varvalue) - varObj._set(varvalue) + varObj._set(varvalue) else: raise DataVarsError( _("Attempt to rewrite a readonly variable")+": %s"%varname) @@ -870,8 +968,10 @@ class DataVars: def Select(self,selField,where="os_disk_dev",eq=None,ne=None, _in=None,like=None,notlike=None,func=None, - sort=None, sortkey=None, limit=None): + sort=None, sortkey=None, limit=None,zipVars=None): """Select value from table variables""" + if zipVars is None: + zipVars = self.zipVars if func: filterFunc = func elif eq != None: @@ -891,13 +991,13 @@ class DataVars: count = len(selField) mapFunc = lambda x:x[1:] res = filter(filterFunc, - self.ZipVars(where,*selField)) + zipVars(where,*selField)) else: count = 1 fields = [where,selField] mapFunc = lambda x:x[1] res = filter(filterFunc, - self.ZipVars(where,selField)) + zipVars(where,selField)) if sort: res.sort(key=sortkey,reverse=True if sort == "DESC" else False) if limit == 1: @@ -1005,7 +1105,6 @@ class DataVars: # invalidate default variables if hasattr(info,"Default") and info.Default: default = info.Default - print "DEFAULT",default for varname in default: self.Invalidate(varname,onlySet=True) else: @@ -1013,12 +1112,15 @@ class DataVars: # look over all vars for var,varObj in map(lambda x:(x.name,x), self._nocheckSort(self._dependSort(varsByKeys))): - # if variable not exists in info data - skip it - if not hasattr(info,var): - continue - # get value of variable from info - val = info.__getattribute__(var) - print var,"=",val.__repr__() + # if info skipped and send None + if info is None: + val = None + else: + # if variable not exists in info data - skip it + if not hasattr(info,var): + continue + # get value of variable from info + val = info.__getattribute__(var) varSendFromClient = not val is None and not var in default varGroupUntrusted = varObj.untrusted and var in groupVars varAllUntrusted = varObj.untrusted and allvars @@ -1030,7 +1132,8 @@ class DataVars: groupIndex = self.mapVarGroup[var] groupVars = list(self.groups[groupIndex]["normal"]) + \ list(self.groups[groupIndex]["expert"]) - if varSendFromClient or varGroupUntrusted or varAllUntrusted: + if varSendFromClient or varGroupUntrusted or varAllUntrusted \ + or not info: try: # get value for final or group check if val is None: @@ -1042,7 +1145,6 @@ class DataVars: raise VariableError(uncomperr) # check and set variable if not self.Uncompatible(var): - print "Set",var,val.__repr__() self.Set(var, val) except VariableError as e: # assemble all variable errors @@ -1050,8 +1152,18 @@ class DataVars: else [e.message] mess = "\n".join(map(lambda x:str(x),messages)) errors.append({'type':'error', 'field':var, 'message':mess}) + except BaseException as e: + for i in apply(traceback.format_exception, sys.exc_info()): + print i + errors.append({'type':'error', 'field':var, + 'message':str(e)}) return errors + def printDependence(self): + for key,val in self.loadVariables.items(): + if val.reqVars: + print key,"->",[x.name for x in val.reqVars] + def printVars(self,filt=lambda x:x): for i in sorted(filter(filt,self.allVars.keys())): print "{0:<40} {1}".format(i,self.Get(i,True)) diff --git a/calculate/lib/utils/device.py b/calculate/lib/utils/device.py index ad77906..0cf821f 100644 --- a/calculate/lib/utils/device.py +++ b/calculate/lib/utils/device.py @@ -220,6 +220,8 @@ def humanreadableSize(size): return "%d.%d%s"%(printSize,printSizeTail,suffix[1]) else: return "%d%s"%(printSize,suffix[1]) + return str(size) + def getPartitionSize(syspath=None,name=None,inBytes=False): """ diff --git a/calculate/lib/utils/ip.py b/calculate/lib/utils/ip.py index 5628aaa..aeb8de9 100644 --- a/calculate/lib/utils/ip.py +++ b/calculate/lib/utils/ip.py @@ -146,10 +146,10 @@ def getIpNet(ip,mask=None,cidr=None): def isIpInNet(checkip,*ipnets): """Check is ip in specified nets""" return map(lambda x:x[0], - filter(lambda x:strIpToIntIp(checkip)&x[2] == strIpToIntIp(x[1])&x[2], - map(lambda x:(x[0],x[1][0],strIpToIntIp(cidrToMask(int(x[1][1])))), - map(lambda x:(x,x.partition('/')[0::2]), - ipnets)))) + filter(lambda x:strIpToIntIp(checkip)&x[2] == strIpToIntIp(x[1])&x[2], + map(lambda x:(x[0],x[1][0],strIpToIntIp(cidrToMask(int(x[1][1])))), + map(lambda x:(x,x.partition('/')[0::2]), + ipnets)))) def isDhcpIp(interface="eth0"): """Get ip by dhcp or static""" diff --git a/calculate/lib/variables/env.py b/calculate/lib/variables/env.py index 69e11d7..d60b8e6 100644 --- a/calculate/lib/variables/env.py +++ b/calculate/lib/variables/env.py @@ -17,7 +17,8 @@ import os import sys from os import path -from calculate.lib.datavars import Variable,ReadonlyVariable,VariableError +from calculate.lib.datavars import (TableVariable,Variable,ReadonlyVariable, + VariableError) from calculate.lib.cl_lang import setLocalTranslate setLocalTranslate('cl_lib3',sys.modules[__name__]) @@ -26,11 +27,10 @@ _envData = [('default', '/etc/calculate/calculate3.env'), ('local', '/var/calculate/calculate3.env'), ('remote', '/var/calculate/remote/calculate3.env')] -class VariableClEnvData(Variable): +class VariableClEnvData(TableVariable): """ Aliases and path to ini files """ - type = "table" source = ["cl_env_location","cl_env_path"] class VariableClEnvLocation(ReadonlyVariable): @@ -68,7 +68,7 @@ class VariableClTemplateCltPath(ReadonlyVariable): """ type = 'list' - def Fill(self): + def get(self): """ Clt templates path is /etc and CONFIG_PROTECT """