calculate
/
calculate-utils-3-lib
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Rewrite table getting. Add uncompatible tracking.

Browse Source
master3.3
Mike Hiretsky 12 years ago
parent 15731a5afc
commit c25b1bb8e0
5 changed files with 330 additions and 213 deletions
Show Stats Download Patch File Download Diff File

7
calculate/lib/cl_template.py
Unescape View File

@ -18,7 +18,10 @@ import sys
import os
import stat
import re
import xml.dom.minidom
import xml.dom.minidom as minidom
import xml
if hasattr(xml,"use_pyxml"):
xml.use_pyxml()
from xml import xpath
import subprocess
import types
@ -1047,7 +1050,7 @@ class xmlDoc:
docTxt += '<ver>%s</ver>'% version
docTxt += '<format>%s</format>' % typeDoc
docTxt += '</head><body></body></cxmlconf>'
self.doc = xml.dom.minidom.parseString(docTxt)
self.doc = minidom.parseString(docTxt)
self.root = self.doc.documentElement
self.body = xpath.Evaluate('child::body',self.root)[0]
# установка разделителя областей

518
calculate/lib/datavars.py
Unescape View File

@ -10,6 +10,7 @@ from threading import RLock
import pyinotify
from itertools import *
import operator
import traceback
def itemgetter(*args,**kwargs):
"""
@ -21,6 +22,14 @@ def itemgetter(*args,**kwargs):
return lambda x:(x[args[0]],)
return operator.itemgetter(*args)
def not_empty(itervalue):
"""
Check has iterator one or more element or not
"""
for i in itervalue:
return True
return False
class DataVarsError(Exception):
"""Exception of getting variable values"""
@ -91,17 +100,18 @@ class Variable:
check_after = []
# fill value is true
untrusted = False
NONE,CHECK,UNCOMPAT,HUMAN = range(4)
def __init__(self,parent=None,**kwargs):
"""
Initialize instance
"""
self.checking = False
self.modeGet = Variable.NONE
self.invalid = self.value is None
self.wasSet = False # variable is user or ini set
self.reqVars = [] # varaibles which used this value for fill
self.reqCheck = [] # variables which used this value for check
self.reqUncompat = [] # varaibles which used this value for uncompat
self.reqUncompat = [] # varaibles which used for uncompat
self.variableType = type
self.parent = parent
self.name = self.getVariableName()
@ -111,7 +121,7 @@ class Variable:
def init(self):
"""
Overridible init
Overridible init using for palce translatable strings
"""
pass
@ -121,13 +131,29 @@ class Variable:
def Get(self,varname=None,humanreadable=False):
"""Get value of other value"""
if not varname:
varname = self.name
if not varname or varname == self.name:
try:
oldmode = self.modeGet
self.modeGet = Variable.NONE
return self._get()
finally:
self.modeGet = oldmode
varObj = self.parent.getInfo(varname)
#varObj.modeGet = Variable.UNCOMPAT
res = self.parent.Get(varname,humanreadable)
if self.checking:
varObj = self.parent.getInfo(varname)
if not self in varObj.reqCheck:
varObj.reqCheck.append(self)
# build dependence for check and uncompatible
if varObj:
if self.modeGet == Variable.NONE:
if not self in varObj.reqVars:
varObj.reqVars.append(self)
if self.modeGet == Variable.CHECK:
if not self in varObj.reqCheck:
varObj.reqCheck.append(self)
elif self.modeGet == Variable.UNCOMPAT:
if not varObj in self.reqUncompat:
self.reqUncompat.append(varObj)
return res
def Choice(self,varname):
@ -136,13 +162,13 @@ class Variable:
def Select(self,selField,**kwargs):
"""Select by datavars"""
return self.parent.Select(selField,**kwargs)
return self.parent.Select(selField,zipVars=self.ZipVars,**kwargs)
def ZipVars(self,*argvVarNames):
"""
Get zipped values of variables specified by list 'argvVarNames'
"""
return self.parent.ZipVars(*argvVarNames)
return zip(*map(self.Get,argvVarNames))
def get(self):
"""
@ -153,7 +179,23 @@ class Variable:
def humanReadable(self):
"""Return human readable value"""
return self.Get(self.name)
return self.Get()
def _human(self):
try:
self.modeGet = Variable.HUMAN
return self.humanReadable()
finally:
self.modeGet = Variable.NONE
def getHumanReadableAuto(self):
"""
Return humanreadable values for readonly variables
"""
if self.mode == READONLY:
return self._human()
else:
return self._get()
def _get(self):
"""
@ -182,6 +224,16 @@ class Variable:
"""
return []
def _choice(self):
"""
Convert choice created by variable choice to two tuple (values,comments)
"""
res = self.choice()
if res and type(res[0]) in (tuple,list):
return zip(*res)
else:
return (res,None)
def check(self,value):
"""
Overridable
@ -199,12 +251,25 @@ class Variable:
"""
return ""
def _uncompatible(self):
"""
Full check uncompatible
"""
try:
self.modeGet = Variable.UNCOMPAT
self.reqUncompat = []
return self.uncompatible()
finally:
self.modeGet = Variable.NONE
def set(self,value):
"""
Overridable
Using for replace value before set
"""
if self.type == "bool":
value = "on" if self.isTrue(value) else "off"
return value
def checkType(self,value):
@ -217,10 +282,11 @@ class Variable:
vartype="list"))
error = _("Values for {varname} may be {vartype} only")
else:
value = [value]
value = repeat(value,1)
error = _("Value for {varname} may be {vartype} only")
if "string" in self.type:
if filter(lambda x:not type(x) in (str,unicode), value):
value, valuecopy = tee(value,2)
for v in (x for x in valuecopy if not type(x) in (str,unicode)):
raise VariableError(error.format(
varname=self.label or self.name,
vartype="string"))
@ -228,7 +294,7 @@ class Variable:
if "choice" in self.type:
choiceVal = self.choice()
if choiceVal and type(choiceVal[0]) in (tuple,list):
choiceVal = map(lambda x:x[0],choiceVal)
choiceVal = [x[0] for x in choiceVal]
for val in value:
if "choice" in self.type and not "choiceedit" in self.type:
if choiceVal and val and not val in choiceVal:
@ -237,39 +303,47 @@ class Variable:
varname=name,
vartype=formatListOr(choiceVal)))
def setValue(self,value):
"""
Standard action for set value
"""
self.value = self.set(value)
self.wasSet = True
self.invalid = False
# run check
self._check()
def _set(self,value,force=False):
"""
Standard inner method for setting value for variable.
"""
# runc check choice
try:
self.checking = True
self.modeGet = Variable.CHECK
self.checkType(value)
finally:
self.checking = False
self.modeGet = Variable.NONE
# if value change
if value != self.value:
self.invalidate(True)
if hasattr(self.value,"close"):
self.value.close()
# run set value
self.value = self.set(value)
self.wasSet = True
self.invalid = False
# run check
self.checkValue()
self.setValue(value)
def checkValue(self):
def _check(self,value=None):
"""
Full check value
"""
if value is None:
value = self.Get(self.name)
try:
self.checking = True
self.modeGet = Variable.CHECK
self.untrusted = True
self.check(self.value)
self.check(value)
self.untrusted = False
finally:
self.checking = False
self.modeGet = Variable.NONE
def invalidate(self,force=False):
"""
@ -289,16 +363,165 @@ class Variable:
self.reqCheck = []
def isTrue(self,value):
if value.lower() in ('yes','on','true'):
if type(value) == bool:
return value
if value.lower() in ('yes','on','true'):
return True
return False
class TableVariable(Variable):
"""
Table type variable
"""
type = "table"
def get(self,hr=False):
"""Get table data"""
for varname,value in ifilter(lambda x:type(x[1]) != list,
imap(lambda x:(x,self.Get(x)),
self.source)):
raise VariableError(
_("Source variable %s not contains list")%varname)
return map(list,
izip_longest(
*map(lambda x:self.Get(x,humanreadable=hr),
self.source),fillvalue='')) or [[]]
def getHumanReadableAuto(self):
"""
Return humanreadable values for readonly variables
"""
return self.get(hr=None)
def humanReadable(self):
return self.get(hr=True)
def __getWritableColumns(self,includeFirst=False):
"""
Get data for writable columns exclude index column
(Example: (1,'os_disk_mount)
"""
if includeFirst:
offset = 0
else:
offset = 1
return filter(lambda x:self.parent.getInfo(x[1]).mode == WRITEABLE,
enumerate(self.source[offset:]))
def check(self,value):
"""
Check table value - check all writeable columns.
"""
writeCols = self.__getWritableColumns(includeFirst=True)
error = []
if not any(value):
value = [[]]*len(writeCols)
else:
value = zip(*map(itemgetter(*map(itemgetter(0),
writeCols),alwaysTuple=True),value))
for colInfo, values in \
zip(writeCols,value):
try:
self.parent.Check(colInfo[1],values)
except VariableError as e:
error.append(e)
if error:
raise VariableError(error)
def __isIndexWritable(self):
return self.parent.getInfo(self.source[0]).mode == WRITEABLE
def checkType(self,value):
"""Check table value type"""
# check type value (list and each element is list)
if not type(value) in (list,tuple) or \
not_empty((i for i in value if not type(i) in (tuple,list))):
raise VariableError(
_("Value for {varname} may be '{vartype}' only").format(
varname=self.label or self.name,
vartype="table"))
# check len all entries
writeLen = len(self.__getWritableColumns())+1
for item in value:
if item and len(item) != writeLen:
raise DataVarsError(
_("Wrong entry '{entry}' for table variable '{varname}'").
format(entry=str(item),
varname=(self.label or self.name).lower()))
# check rewrite readonly index
if not self.__isIndexWritable():
indexValues = self.Get(self.source[0])
for item in value:
if item and not item[0] in indexValues:
raise DataVarsError(
_("Attempt to rewrite a readonly index field "
"{fieldname} in variable {variablename}").format(
fieldname=self.source[0],variablename=self.name))
def setValue(self,value):
self.untrusted = True
oldvalue = self.Get(self.name)
# get writable columns
writeCols = self.__getWritableColumns()
# get slicer
_slice = itemgetter(*map(itemgetter(0),writeCols),
alwaysTuple=True)
# create dict for writable columns
# if table not empty
if any(oldvalue):
oldvalue = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
oldvalue))
else:
oldvalue = OrderedDict()
# get new dict
if any(value):
if len(value[0]) == len(self.source):
newval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
value))
else:
newval = OrderedDict(map(lambda x:(x[0],x[1:]),value))
else:
newval = OrderedDict()
# if table with writable index field then replace all table
error = []
if self.__isIndexWritable():
if any(value):
self.parent.Check(self.source[0],zip(*value)[0])
oldvalue = newval
try:
self.parent.Set(self.source[0],oldvalue.keys())
except VariableError,e:
error.append(e)
# update entry by index field
else:
oldvalue.update(newval)
oldvalValues = zip(*oldvalue.values())
for col,vals in zip(map(lambda x:x[1],writeCols),
oldvalValues):
try:
self.parent.Set(col,list(vals))
except VariableError,e:
error.append(e)
for num,varname in writeCols[len(oldvalValues):]:
self.parent.Invalidate(varname,True)
if error:
raise VariableError(error)
else:
self.untrusted = False
class ReadonlyVariable(Variable):
"""
Alias for readonly variables
"""
mode = READONLY
class ReadonlyTableVariable(TableVariable):
"""
Alias for readonly table
"""
mode = READONLY
class FileVariable(Variable):
"""
Test variable
@ -344,6 +567,9 @@ class SimpleDataVars:
self.cache[varname] = self.allVars.get(varname).get()
return self.cache[varname]
def getInfo(self,varname):
return self.allVars.get(varname, None)
def flIniFileFrom(self,iniFile):
"""
Read variable values from ini files.
@ -555,30 +781,22 @@ class DataVars:
varObj = self.loadVariables[varname]
# if for this variable already use Get method
if varObj.invalid and varObj in self.requestVariables:
varnames = "-".join(map(lambda x:x.name,self.requestVariables))
varnames = "-".join(map(lambda x:x.name,
self.requestVariables+[varObj]))
raise DataVarsError(
_("Loop dependence of variables '%s'")%varnames)
# update dependence variable list for get variable
if self.requestVariables and not self.requestVariables[-1] in \
varObj.reqVars:
varObj.reqVars.append(self.requestVariables[-1])
# add this variable for list of requested variables
if humanreadable is None and varObj.mode == 'r':
humanreadable = True
if not humanreadable:
self.requestVariables.append(varObj)
if varObj.source and Variable.get.__func__ == varObj.get.__func__:
retVal = self.__getTable(varObj,humanreadable)
if humanreadable is None:
res = varObj.getHumanReadableAuto()
elif humanreadable is True:
res = varObj._human()
else:
if humanreadable:
retVal = varObj.humanReadable()
else:
retVal = varObj._get()
# after finish get value - pop variable from request list
res = varObj._get()
if not humanreadable:
self.requestVariables.pop()
#print "Variable(%s)=%s"%(varname,time.time()-tm)
return retVal
return res
except BaseException,e:
while(len(self.requestVariables)):
self.requestVariables.pop()
@ -592,10 +810,7 @@ class DataVars:
if not varname in self.loadVariables:
self.loadVariable(varname)
varObj = self.loadVariables[varname]
if varObj.source and Variable.get.__func__ == varObj.get.__func__:
self.__checkTable(varObj, value)
else:
varObj.checkValue()
varObj._check(value)
def Choice(self, varname):
"""
@ -605,7 +820,7 @@ class DataVars:
if not varname in self.loadVariables:
self.loadVariable(varname)
varObj = self.loadVariables[varname]
return varObj.choice()
return varObj._choice()[0]
def Uncompatible(self, varname):
"""
@ -617,139 +832,32 @@ class DataVars:
if not varname in self.loadVariables:
self.loadVariable(varname)
varObj = self.loadVariables[varname]
return varObj.uncompatible()
return varObj._uncompatible()
def getRequired(self,varname):
"""
Generate list variables using in uncompatible check
"""
added = []
def genReqs(varObj):
if not varObj in added:
yield varObj
for var in varObj.reqVars:
for dep in genReqs(var):
yield dep
if hasattr(varObj,"source") and varObj.source:
for varsource in imap(self.getInfo,varObj.source):
for dep in genReqs(varsource):
yield dep
for dep in genReqs(self.getInfo(varname)):
added.append(dep)
return added
def ChoiceAndComments(self,varname):
choiceVal = self.Choice(varname)
if choiceVal and type(choiceVal[0]) in (tuple,list):
return zip(*choiceVal)
else:
return (choiceVal,None)
def __getTable(self,varobj,hr=False):
def setLen(lst):
if len(lst) < maxlen:
return lst+['']*(maxlen-len(lst))
else:
return lst
maxlen = len(max(map(self.Get,varobj.source),key=len))
wrongValues = filter(lambda x:type(x[1]) != list,
map(lambda x:(x,self.Get(x)),
varobj.source))
if wrongValues:
raise DataVarsError(
_("Variable %s not contains list")%wrongValues[0][0])
return map(lambda x:list(x),
zip(*map(setLen,
map(lambda x:self.Get(x,hr),
varobj.source)))) or [[]]
def __checkTable(self,varObj,value):
"""
Check table value - check all writeable columns.
"""
self.__loadTableVariable(varObj)
writeCols = self.__getWritableColumns(varObj,includeFirst=True)
error = []
if not any(value):
value = [[]]*len(writeCols)
for colInfo, values in \
zip(writeCols,
zip(*map(itemgetter(*map(itemgetter(0),
writeCols),alwaysTuple=True),value))):
try:
self.Check(colInfo[1],values)
except VariableError as e:
error.append(e)
if error:
raise VariableError(error)
def __loadTableVariable(self,varObj):
"""
Load all variables using in table
"""
map(self.loadVariable,
ifilterfalse(self.loadVariables.__contains__,
varObj.source))
def __getWritableColumns(self,varObj,includeFirst=False):
"""
Get data for writable columns exclude index column
(Example: (1,'os_disk_mount)
"""
if includeFirst:
offset = 0
else:
offset = 1
return filter(lambda x:self.loadVariables[x[1]].mode == WRITEABLE,
enumerate(varObj.source[offset:]))
def __isIndexWritable(self,varObj):
return self.loadVariables[varObj.source[0]].mode == WRITEABLE
def __setTable(self,varobj,varvalue):
"""Set variable value to source table variable"""
# get current values
oldval = self.__Get(varobj.name)
# load column variables
self.__loadTableVariable(varobj)
# get writable columns
writeCols = self.__getWritableColumns(varobj)
# get slicer
_slice = itemgetter(*map(itemgetter(0),writeCols),
alwaysTuple=True)
# create dict for writable columns
# if table not empty
if any(oldval):
oldval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
oldval))
else:
oldval = OrderedDict()
# get new dict
if any(varvalue):
if len(varvalue[0]) == len(varobj.source):
newval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
varvalue))
else:
newval = OrderedDict(map(lambda x:(x[0],x[1:]),
varvalue))
else:
newval = OrderedDict()
for item in newval.values():
if len(item) > len(writeCols):
raise DataVarsError(
_("Wrong entry '{entry}' for table variable {varname}").
format(entry=item,varname=varobj.name))
# if table with writable index field then replace all table
error = []
if self.__isIndexWritable(varobj):
if any(varvalue):
self.Check(varobj.source[0],zip(*varvalue)[0])
oldval = newval
try:
self.Set(varobj.source[0],oldval.keys())
except VariableError,e:
error.append(e)
# update entry by index field
else:
if not set(newval.keys()) <= set(oldval.keys()):
raise DataVarsError(
_("Attempt to rewrite a readonly index field "
"{fieldname} in variable {variablename}").format(
fieldname=varobj.source[0],variablename=varobj.name))
oldval.update(newval)
oldvalValues = zip(*oldval.values())
for col,vals in zip(map(lambda x:x[1],writeCols),
oldvalValues):
try:
self.Set(col,list(vals))
except VariableError,e:
error.append(e)
for num,varname in writeCols[len(oldvalValues):]:
self.Invalidate(varname,True)
if error:
raise VariableError(error)
if not varname in self.loadVariables:
self.loadVariable(varname)
varObj = self.loadVariables[varname]
return varObj._choice()
def getInfo(self, varname):
"""
@ -763,19 +871,9 @@ class DataVars:
"""
Set value 'varvalue' for variable 'varname'
"""
if not varname in self.loadVariables:
self.loadVariable(varname)
varObj = self.loadVariables[varname]
varObj = self.getInfo(varname)
if force or varObj.mode != READONLY:
if varObj.source:
self.__setTable(varObj,varvalue)
# check on overriding method (variable has own fill)
if Variable.get.__func__ != varObj.get.__func__:
varObj._set(varvalue)
else:
if varObj.type == "bool" and type(varvalue) == bool:
varvalue = {True:'on',False:'off'}.get(varvalue)
varObj._set(varvalue)
varObj._set(varvalue)
else:
raise DataVarsError(
_("Attempt to rewrite a readonly variable")+": %s"%varname)
@ -870,8 +968,10 @@ class DataVars:
def Select(self,selField,where="os_disk_dev",eq=None,ne=None,
_in=None,like=None,notlike=None,func=None,
sort=None, sortkey=None, limit=None):
sort=None, sortkey=None, limit=None,zipVars=None):
"""Select value from table variables"""
if zipVars is None:
zipVars = self.zipVars
if func:
filterFunc = func
elif eq != None:
@ -891,13 +991,13 @@ class DataVars:
count = len(selField)
mapFunc = lambda x:x[1:]
res = filter(filterFunc,
self.ZipVars(where,*selField))
zipVars(where,*selField))
else:
count = 1
fields = [where,selField]
mapFunc = lambda x:x[1]
res = filter(filterFunc,
self.ZipVars(where,selField))
zipVars(where,selField))
if sort:
res.sort(key=sortkey,reverse=True if sort == "DESC" else False)
if limit == 1:
@ -1005,7 +1105,6 @@ class DataVars:
# invalidate default variables
if hasattr(info,"Default") and info.Default:
default = info.Default
print "DEFAULT",default
for varname in default:
self.Invalidate(varname,onlySet=True)
else:
@ -1013,12 +1112,15 @@ class DataVars:
# look over all vars
for var,varObj in map(lambda x:(x.name,x),
self._nocheckSort(self._dependSort(varsByKeys))):
# if variable not exists in info data - skip it
if not hasattr(info,var):
continue
# get value of variable from info
val = info.__getattribute__(var)
print var,"=",val.__repr__()
# if info skipped and send None
if info is None:
val = None
else:
# if variable not exists in info data - skip it
if not hasattr(info,var):
continue
# get value of variable from info
val = info.__getattribute__(var)
varSendFromClient = not val is None and not var in default
varGroupUntrusted = varObj.untrusted and var in groupVars
varAllUntrusted = varObj.untrusted and allvars
@ -1030,7 +1132,8 @@ class DataVars:
groupIndex = self.mapVarGroup[var]
groupVars = list(self.groups[groupIndex]["normal"]) + \
list(self.groups[groupIndex]["expert"])
if varSendFromClient or varGroupUntrusted or varAllUntrusted:
if varSendFromClient or varGroupUntrusted or varAllUntrusted \
or not info:
try:
# get value for final or group check
if val is None:
@ -1042,7 +1145,6 @@ class DataVars:
raise VariableError(uncomperr)
# check and set variable
if not self.Uncompatible(var):
print "Set",var,val.__repr__()
self.Set(var, val)
except VariableError as e:
# assemble all variable errors
@ -1050,8 +1152,18 @@ class DataVars:
else [e.message]
mess = "\n".join(map(lambda x:str(x),messages))
errors.append({'type':'error', 'field':var, 'message':mess})
except BaseException as e:
for i in apply(traceback.format_exception, sys.exc_info()):
print i
errors.append({'type':'error', 'field':var,
'message':str(e)})
return errors
def printDependence(self):
for key,val in self.loadVariables.items():
if val.reqVars:
print key,"->",[x.name for x in val.reqVars]
def printVars(self,filt=lambda x:x):
for i in sorted(filter(filt,self.allVars.keys())):
print "{0:<40} {1}".format(i,self.Get(i,True))

2
calculate/lib/utils/device.py
Unescape View File

@ -220,6 +220,8 @@ def humanreadableSize(size):
return "%d.%d%s"%(printSize,printSizeTail,suffix[1])
else:
return "%d%s"%(printSize,suffix[1])
return str(size)
def getPartitionSize(syspath=None,name=None,inBytes=False):
"""

8
calculate/lib/utils/ip.py
Unescape View File

@ -146,10 +146,10 @@ def getIpNet(ip,mask=None,cidr=None):
def isIpInNet(checkip,*ipnets):
"""Check is ip in specified nets"""
return map(lambda x:x[0],
filter(lambda x:strIpToIntIp(checkip)&x[2] == strIpToIntIp(x[1])&x[2],
map(lambda x:(x[0],x[1][0],strIpToIntIp(cidrToMask(int(x[1][1])))),
map(lambda x:(x,x.partition('/')[0::2]),
ipnets))))
filter(lambda x:strIpToIntIp(checkip)&x[2] == strIpToIntIp(x[1])&x[2],
map(lambda x:(x[0],x[1][0],strIpToIntIp(cidrToMask(int(x[1][1])))),
map(lambda x:(x,x.partition('/')[0::2]),
ipnets))))
def isDhcpIp(interface="eth0"):
"""Get ip by dhcp or static"""

8
calculate/lib/variables/env.py
Unescape View File

@ -17,7 +17,8 @@
import os
import sys
from os import path
from calculate.lib.datavars import Variable,ReadonlyVariable,VariableError
from calculate.lib.datavars import (TableVariable,Variable,ReadonlyVariable,
VariableError)
from calculate.lib.cl_lang import setLocalTranslate
setLocalTranslate('cl_lib3',sys.modules[__name__])
@ -26,11 +27,10 @@ _envData = [('default', '/etc/calculate/calculate3.env'),
('local', '/var/calculate/calculate3.env'),
('remote', '/var/calculate/remote/calculate3.env')]
class VariableClEnvData(Variable):
class VariableClEnvData(TableVariable):
"""
Aliases and path to ini files
"""
type = "table"
source = ["cl_env_location","cl_env_path"]
class VariableClEnvLocation(ReadonlyVariable):
@ -68,7 +68,7 @@ class VariableClTemplateCltPath(ReadonlyVariable):
"""
type = 'list'
def Fill(self):
def get(self):
"""
Clt templates path is /etc and CONFIG_PROTECT
"""

Write Preview
Loading…
Cancel
Save