calculate-utils-3-lib/calculate/lib/datavars.py

#-*- coding: utf-8 -*-

# Copyright 2008-2013 Calculate Ltd. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re
import sys
import importlib
from utils.text import convertStrListDict, _toUNICODE, formatListOr
from utils.files import pathJoin,readFile
from utils.portage import searchProfile
from os import path
import os
from collections import OrderedDict
import time
from threading import RLock
import pyinotify
from itertools import *
import operator
import traceback
from types import StringTypes

def addStdConfig(fileLst,prefix='/'):
    stdPath = path.join(prefix,
              "var/lib/layman/calculate/profiles/calculate.env")
    if not stdPath in map(os.path.abspath,fileLst):
            return  [stdPath] + fileLst
    return fileLst

def itemgetter(*args,**kwargs):
    """
    itemgetter with alwaysTuple param, which used for force tuple elements
    """
    alwaysTuple = kwargs.get('alwaysTuple',False)
    if alwaysTuple:
        if len(args)==1:
            return lambda x:(x[args[0]],)
    return operator.itemgetter(*args)

class CriticalError(Exception):
    """Critical error"""

class DataVarsError(Exception):
    """Exception of getting variable values"""

class VariableError(Exception):
    """Exception of sended by Variable"""

class PasswordError(VariableError):
    """Exception of password change error"""

class CommonVariableError(VariableError):
    """Exception for command line"""

import cl_template

def makePath(dirs,mode=755):
    if not path.exists(dirs):
        os.makedirs(dirs,mode)

from calculate.lib.cl_lang import setLocalTranslate
setLocalTranslate('cl_lib3',sys.modules[__name__])

READONLY = "r"
WRITEABLE = "w"

class Variable:
    """
    Class of variable. Need for creating calculate utilities variable
    add fill method described by class with fill methods.

    Instance attributes:
      wasSet : boolean
        variable value was set by _Set method
      reqVars : List
        list of variable which use this variable in fill method
      invalid : boolean
        invalid value
      section : string
        section of variable (main,install,builder and etc)
    """
    # label of variable
    label = None
    # opts for cmdline
    opt = None
    # metavalues
    metavalue = None
    # help information
    help = None
    # writeable variable
    mode = WRITEABLE
    # hidden variable
    hide = True
    # default value
    value = None
    # variable section
    section = "main"
    # variable name
    name = ""
    # gui label
    label = None
    # syntax for opts
    syntax = ""
    # opt for console
    opt = None
    # help for console
    help = None
    # variable type: string, multichoice, list, number, choiceedit, table
    # flag, yesno, password
    type = "string"
    # metavalue for opt
    metavalue = None
    # source column with data
    source = None
    # gui elelemnt type
    element = None
    # force check after variables in list
    check_after = []
    # fill value is true
    untrusted = False
    NONE,CHECK,UNCOMPAT,HUMAN = range(4)
    # при установке значения переменной устанавливает значение
    # указанной переменной
    alias = ""

    def __init__(self,parent=None,**kwargs):
        """
        Initialize instance
        """
        self.modeGet = Variable.NONE
        self.wasSet = False     # variable is user or ini set
        self.reqVars = []       # varaibles which used this value for fill
        self.reqCheck = []      # variables which used this value for check
        self.reqUncompat = []   # varaibles which used for uncompat
        self.variableType = type
        self.parent = parent
        self.invalid = True # self.value is None or self.name in self.parent.iniCache
        self.name = self.getVariableName()
        self.processInvalidate = False
        # производится чтение из ini
        self.bIniRead = False
        self.init()
        for key,val in kwargs.items():
            setattr(self,key,val)

    def init(self):
        """
        Overridible init using for palce translatable strings
        """
        pass

    @classmethod
    def getVariableName(cls):
        return re.sub("(.)([A-Z])","\\1_\\2",cls.__name__[8:]).lower()

    def getCharType(self):
        """
        Get symbol of type
        """
        mapType = OrderedDict(table='t',list='l',choice='c',number='i',
                              bool='b') 
        for k in mapType.keys():
            if k in self.type:
                return mapType[k]
        else:
            return "s"

    def Get(self,varname=None,humanreadable=False):
        """Get value of other value"""
        if not varname or varname == self.name:
            return self._get()

        varObj = self.parent.getInfo(varname)
        #varObj.modeGet = Variable.UNCOMPAT
        if not "." in varname:
            varname = "%s.%s"%(self.section,varname)
        res = self.parent.Get(varname,humanreadable)

        # build dependence for check and uncompatible
        if varObj:
            if self.modeGet == Variable.NONE:
                if not self in varObj.reqVars:
                    varObj.reqVars.append(self)
            if self.modeGet == Variable.CHECK:
                if not self in varObj.reqCheck:
                    varObj.reqCheck.append(self)
            elif self.modeGet == Variable.UNCOMPAT:
                if not varObj in self.reqUncompat:
                    self.reqUncompat.append(varObj)
        return res

    def Choice(self,varname):
        """Choice from other value"""
        return self.parent.Choice(varname)

    def Select(self,selField,**kwargs):
        """Select by datavars"""
        return self.parent.Select(selField,zipVars=self.ZipVars,**kwargs)

    def getRemoteInfo(self, envFile):
        """getRemoteInfo by datavars"""
        return self.parent.getRemoteInfo(envFile)

    def ZipVars(self,*argvVarNames):
        """
        Get zipped values of variables specified by list 'argvVarNames'
        """
        return zip(*map(self.Get,argvVarNames))

    def get(self):
        """
        Overridable
        Calculate value method
        """
        if self.__class__.value:
            return self.__class__.value
        else:
            return [] if "list" in self.type else ""

    def humanReadable(self):
        """Return human readable value"""
        val = self.Get()
        if "choice" in  self.type:
            choiceVal, commentVal = self._choice()
            if commentVal:
                valDict = dict(zip(choiceVal,commentVal))
                if "list" in self.type:
                    return map(lambda x:valDict.get(x,x),val)
                else:
                    return valDict.get(val,val)
        return val

    def _human(self):
        # TODO: may be rest to NONE
        try:
            oldModeGet = self.modeGet
            self.modeGet = Variable.HUMAN
            return self.humanReadable()
        finally:
            self.modeGet = oldModeGet


    def getHumanReadableAuto(self):
        """
        Return humanreadable values for readonly variables
        """
        if self.mode == READONLY:
            return self._human()
        else:
            return self._get()

    def iniread(self):
        """
        Read from ini file
        """
        if self.bIniRead:
            return False
        self.bIniRead = True
        try:
            value = self.parent.iniCache[self.name]['value']
            section = self.parent.iniCache[self.name]['section']
            value = self.parent.unserialize(self.type,
                                     value.encode('utf-8'))
            if self.mode == READONLY:
                raise VariableError(
                    _("Attempt to rewrite readonly variable %s")%
                    self.name)
            else:
                try:
                    self._set(value)
                except Exception as e:
                    return False
            return True
        finally:
            self.bIniRead = False

    def _get(self):
        """
        Standard inner method for getting value of variable.
        """
        # variable need refill
        if self.invalid:
            # get net value
            if self.name in self.parent.iniCache and \
                self.iniread():
                return self.value
            else:
                try:
                    oldModeGet = self.modeGet
                    self.modeGet = Variable.NONE
                    value = self.get()
                finally:
                    self.modeGet = oldModeGet
            # if value not equal previous value
            if value != self.value:
                # invalidate depended variables
                self.invalidate()
            # mark as filled
            self.invalid = False
            if hasattr(self.value,"close"):
                self.value.close()
            self.value = value
        return self.value

    def choice(self):
        """
        Overridable

        Generation list of allowable values
        """
        return []

    def _choice(self):
        """
        Convert choice created by variable choice to two tuple (values,comments)
        """
        res = self.choice()
        if res and type(res[0]) in (tuple,list):
            return zip(*res)
        else:
            return (res,None)

    def check(self,value):
        """
        Overridable

        Check method, for wrong value need raise VariableError
        """
        pass

    def uncompatible(self):
        """
        Overridable

        Using for check uncompatibility using variable with value of
        other variables.
        """
        return ""

    def _uncompatible(self):
        """
        Full check uncompatible
        """
        try:
            self.modeGet = Variable.UNCOMPAT
            self.reqUncompat = []
            return self.uncompatible()
        finally:
            self.modeGet = Variable.NONE

    def set(self,value):
        """
        Overridable

        Using for replace value before set
        """
        def convert(value):
            if value and value.lower() != "auto":
                return "on" if self.isTrue(value) else "off"
            else:
                return ""
        if self.type == "bool":
            value = "on" if self.isTrue(value) else "off"
        if "bool" in self.type and "list" in self.type:
            return map(convert,value)
        if "int" in self.type:
            return str(value)
        return value

    def raiseNothingValue(self):
        """
        Using for override error message about nothing value.
        """
        pass

    def checkType(self,value):
        """Check value for type"""
        if "int" in self.type:
            if value and not value.isdigit():
                raise VariableError(
                    _("The value of variable '{varname}' must be integer"
                            ).format(varname=self.label or self.name))
        if "list" in self.type:
            if not type(value) in (list,tuple):
                raise VariableError(
                    _("The value for variable '{varname}' may be {vartype} only").format(
                    varname=self.label or self.name,
                    vartype="list"))
            error = _("Values for variable '{varname}' may be {vartype} only")
        else:
            value = repeat(value,1)
            error = _("The value for variable '{varname}' may be {vartype} only")
        if "string" in self.type:
            value, valuecopy = tee(value,2)
            for v in (x for x in valuecopy if not type(x) in (str,unicode)):
                raise VariableError(error.format(
                    varname=self.label or self.name,
                    vartype="string"))
            return
        if "choice" in self.type:
            choiceVal = self.choice()
            tipChoice = map(lambda x:'"%s" [%s]'%(x[1],x[0]) \
                            if type(x) in (list,tuple) else str(x), choiceVal)
            if choiceVal and type(choiceVal[0]) in (tuple,list):
                choiceVal = [x[0] for x in choiceVal]
        if not "list-list" in self.type:
            value = repeat(value,1)
        for recval in value:
            for val in recval:
                name = self.label or self.name
                if val == "list" and "choice" in self.type:
                    if "edit" in self.type:
                        if "list" in self.type:
                            error = _("Values for variable '{varname}' may "
                                      "be {vartype}")
                        else:
                            error = _("The value for variable '{varname}' may be "
                                      "{vartype}")
                    self.raiseWrongChoice(name, tipChoice, val, error)
                if "choice" in self.type and not "choiceedit" in self.type:
                    if choiceVal and val and not val in choiceVal or \
                        val is None:
                        if not choiceVal:
                            self.raiseNothingValue()
                        self.raiseWrongChoice(name,tipChoice,val,error)

    def raiseWrongChoice(self,name,choiceVal,value,error):
        raise VariableError(error.format(
                            varname=name,
                            vartype=formatListOr(choiceVal)))

    def setValue(self,value,force=False):
        """
        Standard action for set value
        """
        self.value = self.set(value)
        self.wasSet  = True
        self.invalid = False
        # run check
        if not force:
            self._check()

    def _set(self,value,force=False):
        """
        Standard inner method for setting value for variable.
        """
        # runc check choice
        if type(value) == int:
            value = str(value)
        if not force:
            try:
                self.modeGet = Variable.CHECK
                self.checkType(value)
            finally:
                self.modeGet = Variable.NONE
        # if value change
        if value != self.value or type(value) in (list,tuple):
            self.invalidate(True)
        if hasattr(self.value,"close"):
            self.value.close()
        # run set value
        self.setValue(value,force)

    def _check(self,value=None):
        """
        Full check value
        """
        if value is None:
            value = self.Get(self.name)
        try:
            self.modeGet = Variable.CHECK
            self.untrusted = True
            self.check(value)
            self.untrusted = False
        finally:
            self.modeGet = Variable.NONE

    def invalidate(self,force=False):
        """
        Invalidate value of variable (drop filled value).

        force : boolean=False
          Drop value also for after manual set
        """
        # fix possible infinite recursion on invalidating
        if not self.processInvalidate and (force or not self.wasSet):
            try:
                # mark variable that in invalidation now
                self.processInvalidate = True
                self.wasSet = False
                self.invalid = True
                for var in self.reqVars:
                    var.invalidate()
                for var in self.reqCheck:
                    var.untrusted = True
                self.reqVars = []
                self.reqCheck = []
            finally:
                # unmark process invalidate
                self.processInvalidate = False

    @classmethod
    def isTrue(self,value):
        if type(value) == bool:
            return value
        if  value.lower() in ('yes','on','true'):
            return True
        return False

class TableVariable(Variable):
    """
    Table type variable
    """
    type = "table"

    def set(self,value):
        return value

    def get(self,hr=False):
        """Get table data"""
        for varname,value in ifilter(lambda x:type(x[1]) != list,
                             imap(lambda x:(x,self.Get(x)),
                             self.source)):
            raise VariableError(
                _("Source variable %s does not contain a list")%varname)
        return map(list,
               izip_longest(
               *map(lambda x:self.Get(x,humanreadable=hr),
               self.source),fillvalue='')) or [[]]

    def getHumanReadableAuto(self):
        """
        Return humanreadable values for readonly variables
        """
        return self.get(hr=None)

    def humanReadable(self):
        return self.get(hr=True)

    def __getWritableColumns(self,includeFirst=False):
        """
        Get data for writable columns exclude index column
        (Example: (1,'os_disk_mount)
        """
        if includeFirst:
            offset = 0
        else:
            offset = 1
        return filter(lambda x:self.parent.getInfo(x[1]).mode == WRITEABLE,
               enumerate(self.source[offset:]))

    def check(self,value):
        """
        Check table value - check all writeable columns.
        """
        writeCols = self.__getWritableColumns(includeFirst=True)
        error = []
        if not any(value):
            value = [[]]*len(writeCols)
        else:
            value = zip(*map(itemgetter(*map(itemgetter(0),
                                writeCols),alwaysTuple=True),value))
        for colInfo, values in \
            zip(writeCols,value):
            try:
                self.parent.Check(colInfo[1],values)
            except VariableError as e:
                error.append(e)
        if error:
            raise VariableError(error)

    def __isIndexWritable(self):
        return self.parent.getInfo(self.source[0]).mode == WRITEABLE

    def checkType(self,value):
        """Check table value type"""
        # check type value (list and each element is list)
        if not type(value) in (list,tuple) or \
            any(i for i in value if not type(i) in (tuple,list)):
            raise VariableError(
                    _("The value for {varname} may be '{vartype}' only").format(
                    varname=self.label or self.name,
                    vartype="table"))
        # check len all entries
        writeLen = len(self.__getWritableColumns())+1
        for item in value:
            if item and not len(item) in (writeLen,len(self.source)):
                raise VariableError(
                    _("Wrong entry '{entry}' for table variable '{varname}'").
                    format(entry=str(item),
                    varname=(self.label or self.name).lower())+"\n"+
                    _("Each entry must contains %d field(s)")%writeLen)

        # check rewrite readonly index
        if not self.__isIndexWritable():
            indexValues = self.Get(self.source[0])
            for item in value:
                if item and not item[0] in indexValues:
                    self.raiseReadonlyIndexError(self.source[0],
                                                 self.name,
                                                 item[0])

    def raiseReadonlyIndexError(self,fieldname="",variablename="",
                                     value=""):
        """
        Overriddible method for spcifing behavior on rewrite
        readonly index
        """
        raise DataVarsError(
            _("Attempt to rewrite a readonly index field "
            "{fieldname} in variable {variablename}").format(
            fieldname=fieldname,variablename=variablename))

    def setValue(self,value,force=False):
        value = self.set(value)
        self.untrusted = True
        oldvalue = self.Get(self.name)
        # get writable columns
        writeCols = self.__getWritableColumns()
        # get slicer
        _slice = itemgetter(*map(itemgetter(0),writeCols),
                            alwaysTuple=True)
        # create dict for writable columns
        # if table not empty
        if any(oldvalue):
            oldvalue = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
                                   oldvalue))
        else:
            oldvalue = OrderedDict()
        # get new dict
        if any(value):
            if len(value[0]) == len(self.source):
                newval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
                                     value))
            else:
                newval = OrderedDict(map(lambda x:(x[0],x[1:]),value))
        else:
            newval = OrderedDict()
        # if table with writable index field then replace all table
        error = []
        if self.__isIndexWritable():
            if not force and any(value):
                self.parent.Check(self.source[0],zip(*value)[0])
            oldvalue = newval
            try:
                self.parent.Set(self.source[0],oldvalue.keys())
            except VariableError,e:
                error.append(e)
        # update entry by index field
        else:
            oldvalue.update(newval)
        oldvalValues = zip(*oldvalue.values())
        for col,vals in zip(map(lambda x:x[1],writeCols),
                            oldvalValues):
            try:
                self.parent.Set(col,list(vals))
            except VariableError,e:
                error.append(e)
        for num,varname in writeCols[len(oldvalValues):]:
            self.parent.Invalidate(varname,True)
        if error:
            raise VariableError(error)
        else:
            self.untrusted = False

class ReadonlyVariable(Variable):
    """
    Alias for readonly variables
    """
    mode = READONLY

class ActionVariable(ReadonlyVariable):
    nonchroot = False
    postinst = False

    def get(self):
        if self.nonchroot and self.Get('cl_chroot_status') == "on":
            return "off"
        if self.postinst and not self.Get('cl_ebuild_phase') in \
            ('','postinst','prerm'):
            return "off"
        return self.action(self.Get('cl_action'))

class FieldValue:
    """
    Table column variable
    """
    type = "list"
    source_variable = ""
    column = 0

    def get(self):
        sourceVar = self.Get(self.source_variable)
        if any(sourceVar):
            return zip(*sourceVar)[self.column]
        else:
            return []

class SourceReadonlyVariable(ReadonlyVariable):
    """
    Source readonly variable (Using for table)
    Example:

    ['os_location_source',...,'os_location_size'(this)]
    """
    indexField = ""

    def get(self):
        return map(lambda x:x or "",
               map(self.getMap().get,
               self.Get(self.indexField)))

    def humanReadable(self):
        return map(lambda x:x or "",
               map(self.getMapHumanReadable().get,
               self.Get(self.indexField)))

    def getMap(self):
        return {}

    def getMapHumanReadable(self):
        return self.getMap()

class ReadonlyTableVariable(TableVariable):
    """
    Alias for readonly table
    """
    mode = READONLY

class FileVariable(Variable):
    """
    Test variable
    """
    filename = ""
    directory = ""
    recursive = False
    wm = None

    def init(self):
        if self.wm is None:
            self.wm = pyinotify.WatchManager()
            if self.filename:
                self.mask = pyinotify.IN_MODIFY
            else:
                self.mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE
            self.notifier = pyinotify.Notifier(self.wm,timeout=1)
            self.wdd = self.wm.add_watch(self.filename or self.directory,
                                         self.mask,
                                         rec=self.recursive)

    def close(self):
        self.notifier.stop()

    def refresh(self):
        if self.notifier.check_events():
            self.notifier.read_events()
            self.parent.Invalidate(self.name)

class SimpleDataVars:
    """
    Simple datavars for temporary link var objs.
    """
    systemRoot = "/"
    def __init__(self,*objs):
        self.allVars = {}
        self.cache = {}
        for obj in objs:
            obj.parent = self
            self.allVars[obj.getVariableName()] = obj

    def Get(self,varname,humanreadable=False):
        varname = varname.rpartition(".")[2]
        if not varname in self.cache:
            if self.allVars.get(varname) is not None:
                self.cache[varname] = self.allVars.get(varname).get()
            else:
                raise DataVarsError(_("Failed to get value %s")%varname)
        return self.cache[varname]

    def getInfo(self,varname):
        return self.allVars.get(varname, None)

    def flIniFileFrom(self,iniFile):
        """
        Read variable values from ini files.
        Return False if ini file is invalid.
        """
        if path.isdir(iniFile):
            config = cl_template.iniParser(
                text=reduce(lambda x, y: x + readFile(y),
                            addStdConfig(
                                searchProfile(iniFile, 'calculate.env'),
                                prefix=self.systemRoot), ""))
        else:
            config = cl_template.iniParser(iniFile)
        importVars = {}
        for varname, varobj in self.allVars.items():
            if varobj.section not in importVars:
                importVars[varobj.section] = config.getAreaVars(varobj.section)
            if not varname in self.cache:
                val = self.unserialize(varobj.type or "string",
                    importVars[varobj.section].get(varname,'').encode('utf-8'))
                if val or varname in importVars[varobj.section]:
                    self.cache[varname] = val

    @staticmethod
    def serialize(varType, value):
        """
        Serialize to string for ini file
        """
        fixEmpty = lambda x: str(x) or "''"
        fixSpace = lambda x: "'%s'" % str(x) if " " in str(x) else str(x)
        isListOrTuple = lambda x: type(x) in (list, tuple)
        if isListOrTuple(value):
            if "list" in varType:
                return ",".join(map(fixEmpty, value))
            elif "table" in varType:
                return ",".join(map(lambda x: ":".join(map(fixEmpty, x)),
                                    value))
        return fixEmpty(value)

    @staticmethod
    def unserialize(varType, value):
        """
        Unserialize form string for varname
        """
        fixEmpty = lambda x: "" if x == "''" or x == '""' else x.strip()

        def getList(delimeter=','):
            def wrapper(val):
                if val == "":
                    return []
                return map(fixEmpty, val.split(delimeter))

            return wrapper

        if "list" in varType:
            return getList()(value)
        if "table" in varType:
            return map(getList(':'), value.split(','))
        return fixEmpty(value).strip("'").strip('"')


class DataVars(SimpleDataVars):
    """
    Class contains variables, perform creating variable objects by
    class variable describer and variable filler.

    Instance attributes:
      moduleList : Dict
        dictionary has section name with modules which describe variable
        for this sectin
      loadVariables : Dict
        contains all initialized variables load from varObj and fillObj
      requestVariables : List
        contains var object which already run Get method. Use for detect
        dependence of variables.
    """
    defaultData = 'calculate.lib.variables'
    reRightVar = re.compile(r'^(?:([a-z_]+)\.)?([a-z_0-9]+)$')
    l = RLock()

    def __init__(self):
        self.requestVariables = []
        self.loadVariables = {}
        self.allVars = {}
        self.untrustedVariables = set()
        self.refresh = []
        self.groups = []
        self.mapVarGroup = OrderedDict()
        self.briefData = {}
        self.filledVars = {}
        self.defaultModule = "main"
        self.importedModules = []
        self.iniCache = {}

    def isModuleInstalled(self,module):
        try:
            importlib.import_module("calculate.%s.datavars"%module)
            return True
        except (ImportError) as e:
            return False

    def importData(self,data=None):
        """
        Depricated function for support 3.0 utilities
        """
        section = self.importVariables(data)
        if data:
            self.defaultModule = section

    def importVariables(self,data=None):
        """
        Import modules for variables. 'section' is section name, 'modList'
        contains tuple with modules which discribe variable names and fill
        methods
        """
        # try import vars and fill modules specified by modList
        if not data:
            data = self.defaultData
        try:
            varModule = importlib.import_module(data)
        except (ImportError,AttributeError),e:
            if not self.raiseModuleError(data,e):
                return False
        # get all Variable classes from module
        # variable of class VariableClAction will be named cl_action
        if hasattr(varModule,"section"):
            section = varModule.section
        else:
            section = "main"
        for varMod in map(lambda x:getattr(varModule,x),
                      filter(lambda x:not x.startswith("_") and \
                             not "Variable" in x,
                      dir(varModule)))+[varModule]:
            for classname in \
                    ifilterfalse(("ReadonlyVariable",
                                 "Variable","VariableError").__contains__,
                    ifilter(lambda x:x.startswith("Variable"),
                    dir(varMod))):
                varObj = getattr(varMod,classname)
                varName = varObj.getVariableName()
                if not varName in self.allVars:
                    self.allVars[varName] = (section,varObj)
        self.importedModules.append(section)
        return section

    def raiseVariableNotFound(self,varname,*args,**kwargs):
        raise DataVarsError(_("Variable %s not found")%varname)

    def raiseModuleError(self,module,error,*args,**kwargs):
        raise DataVarsError("\n".join([
            _("Failed to import module %s")%module,
            _("error")  + ": " +str(error)]))

    def loadVariable(self,varname,section=None):
        """
        Initialize variable 'varname' from fillObj and varObj.
        """
        if not self.allVars:
            self.importVariables()
        if section and not section in self.importedModules:
            self.importVariables("calculate.%s.variables"%section)
            self.flIniFile(onlySection=section)
        if not varname in self.loadVariables:
            if varname in self.allVars.keys():
                section,varObj = self.allVars[varname]
                newVar = varObj(parent=self,section=section)
                if hasattr(newVar,"refresh"):
                    self.refresh.append(newVar)
                self.loadVariables[varname] = newVar
            else:
                # this code using for raise exception or
                # if replace raiseVariableNotFound method
                # then return default value
                self.loadVariables[varname] = \
                    self.raiseVariableNotFound(
                        varname,parent=self,section="main")

    def exists(self,varName):
        try:
            self.Get(varName)
            return True
        except:
            return False

    def defined(self,varName):
        """
        Return exists or not varName
        """
        return varName in self.allVars.keys()

    def getIniVar(self,fullVarName):
        """
        Get variable values from ini files. Variable specified by
        'fullVarName' in format: section 'dot' variable. If specified
        only variable then section will be 'main'

        Example:
        self.getIniVar('install.os_install_net_settings')
        self.getIniVar('cl_action')
        """
        calculateIniFiles = self.Get('main.cl_env_path')
        section, dot, varname = fullVarName.rpartition('.')
        if not section:
            section = "main"
        retVal = ""
        for iniFile in calculateIniFiles:
            if path.exists(iniFile):
                config = cl_template.iniParser(iniFile)
                data = config.getVar(section,varname,checkExistVar=True)
                if data is False:
                    return False
                existsVar, value = data
                if existsVar:
                    retVal = value
        return retVal.encode('UTF-8')

    def getRemoteInfo(self, envFile):
        """
        Get information variables from envFile
        Return dict:
        section : { variable1 : value,
                    variable2 : value }
        section2 : { variable : value }
        """
        optionsInfo = {}
        config = cl_template.iniParser(envFile)
        allsect = config.getAllSectionNames()
        if allsect:
            for section in allsect:
                allvars = config.getAreaVars(section)
                if allvars == False:
                    return False
                # options (dict of variables:value)
                options = {}
                for varName, value in allvars.items():
                    varName = varName.encode("UTF-8")
                    value=convertStrListDict(value.encode("UTF-8"))
                    options[varName] = value
                # if options exists add section to return dict as key section
                if options:
                    optionsInfo[section] = options
        return optionsInfo

    def flIniFile(self,raiseOnError=False,onlySection=None):
        """
        Read variable values from ini files.
        Return False if ini file is invalid.
        """
        if not self.iniCache:
            # get initialized section names
            try:
                make_profile = self.Get('main.cl_make_profile')
            except VariableError as e:
                action = self.Get('cl_action')
                if not action or action in ['update_profile']:
                    return True
                raise CriticalError(str(e))
            if os.path.exists(make_profile):
                profiles = [('profile',
                            path.join('/etc',os.readlink(make_profile)))]
            else:
                profiles = []
            for iniName,iniFile in profiles+\
                                    self.Get('main.cl_env_data'):
                if path.exists(iniFile) or iniName == 'profile':
                    if iniName == 'profile':
                        config = cl_template.iniParser(
                            text=reduce(lambda x,y:x+readFile(y),
                            addStdConfig(
                            searchProfile(iniFile,'calculate.env')),""))
                    else:
                        config = cl_template.iniParser(iniFile)
                    iniSections = config.getAllSectionNames()
                    if not iniSections:
                        continue
                    if onlySection:
                        iniSections = tuple(set(iniSections)&
                                            set([unicode(onlySection)]))
                    for section in iniSections:
                        importVars = config.getAreaVars(section)
                        if self.allVars == False:
                            continue
                        for key,value in importVars.items():
                            key = _toUNICODE(key).encode('utf-8')
                            self.iniCache[key] = {'value':value,
                                              'section':section}
                            if iniName != "profile":
                                self.filledVars[key] = iniName
        return True

    def Get(self,varname,humanreadable=False):
        """Threading safety Get"""
        DataVars.l.acquire()
        try:
            var = self.__Get(varname,humanreadable)
        finally:
            DataVars.l.release()
        return var

    def GetBool(self,varname):
        return Variable.isTrue(self.Get(varname))

    def GetInteger(self,varname):
        """
        Получить целочисленное значение
        """
        try:
            return int(self.Get(varname))
        except ValueError:
            return 0

    def isFromIni(self,varname):
        """
        Check what value get from ini file
        """
        section,varname = self.splitVarname(varname)
        return self.filledVars[varname] if varname  in self.filledVars else ""

    def splitVarname(self,varname):
        res = self.reRightVar.search(varname)
        if res:
            if res.group(1):
                return res.groups()
            else:
                return (self.defaultModule,res.group(2))
        else:
            raise DataVarsError(_("Wrong variable name %s")%varname)

    def __Get(self, varname, humanreadable=False):
        """
        Get value of variable 'varname'
        """
        try:
            #tm = time.time()
            # load variable if it isn't loaded
            section,varname = self.splitVarname(varname)
            if not varname in self.loadVariables or \
                not section in self.importedModules:
                self.loadVariable(varname,section=section)
            varObj = self.loadVariables[varname]

            # check section
            if not varObj.section in (section,"main"):
                self.raiseVariableNotFound(varname,parent=self,section=section)

            # if for this variable already use Get method
            if varObj.invalid and varObj in self.requestVariables:
                varnames = "-".join(map(lambda x:x.name,
                                    self.requestVariables+[varObj]))
                raise DataVarsError(
                    _("Loop dependence of variables '%s'")%varnames)
            # add this variable for list of requested variables
            if not humanreadable:
                self.requestVariables.append(varObj)
            if humanreadable is None:
                res = varObj.getHumanReadableAuto()
            elif humanreadable is True:
                res = varObj._human()
            else:
                res = varObj._get()
            if not humanreadable:
                if self.requestVariables:
                    self.requestVariables.pop()
            return res
        except BaseException,e:
            while(len(self.requestVariables)):
                self.requestVariables.pop()
            raise

    def Check(self,varname,value):
        """
        Check varialbe value
        """
        # load variable if it isn't loaded
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        varObj.checkType(value)
        varObj._check(value)

    def Choice(self, varname):
        """
        Get choice value of variable 'varname'
        """
        # load variable if it isn't loaded
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        return varObj._choice()[0]

    def Uncompatible(self, varname):
        """
        Check on compatible change value of this variable

        Empty string if variable changable or string with error
        """
        # load variable if it isn't loaded
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        return varObj._uncompatible()

    def getRequired(self,varname):
        """
        Generate list variables using in uncompatible check
        """
        added = []
        def genReqs(varObj):
            if not varObj in added:
                yield varObj
                for var in varObj.reqVars:
                    for dep in genReqs(var):
                        yield dep
                if hasattr(varObj,"source") and varObj.source:
                    for varsource in imap(self.getInfo,varObj.source):
                        for dep in genReqs(varsource):
                            yield dep
        for dep in genReqs(self.getInfo(varname)):
            added.append(dep)
        return added

    def ChoiceAndComments(self,varname):
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        return varObj._choice()

    def getInfo(self, varname):
        """
        Get info (label and other for variable)
        """
        section,varname = self.splitVarname(varname)
        if not varname in self.loadVariables:
            self.loadVariable(varname,section=section or None)
        return self.loadVariables[varname]

    def Set(self, varname, varvalue,force=False):
        """
        Set value 'varvalue' for variable 'varname'
        """
        varObj = self.getInfo(varname)
        if force or varObj.mode != READONLY:
            varObj._set(varvalue,force)
            if varObj.alias:
                self.Set(varObj.alias,varObj.value,force)
        else:
            raise DataVarsError(
                _("Attempt to rewrite readonly variable %s")%
                varname)

    def Invalidate(self,varname,force=False,onlySet=False):
        """
        Invalidate value of variable. Force - invalidate also for set values
        """
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        if not onlySet or varObj.wasSet:
            varObj.invalidate(force or onlySet)

    def getEnvFileByLocation(self,location):
        """
        Get env file path by location alias. This path append to value
        of variable 'cl_chroot_path'.
        Return False if alias not found.
        """
        calculateIniFile = self.Select('cl_env_path',
                                       where='cl_env_location',
                                       eq=location,limit=1)
        if calculateIniFile:
            return pathJoin(self.Get('main.cl_chroot_path'),calculateIniFile)
        else:
            return False

    def Write(self, varname, val=None, force=False, location='system',
              header=False):
        """
        Write variable 'varname' to ini file. If set 'val', then to
        variable before write perform value changing. 'force' use for
        change value of read-only variables. 'location' must be ('system',
        'local', 'remote') use for specify type of ini file location ini.
        'header' use for specify section in ini file. If header not
        specified then use default for this value section.
        """
        # set value to variable if specified value
        if not val is None:
            self.Set(varname,val,force=force)
        else:
            if header:
                vname = "%s.%s"%(header,varname)
            val = self.Get(vname)
        # get path to target ini file
        iniFile = self.getEnvFileByLocation(location)
        if iniFile:
            # make parent directories
            makePath(path.dirname(iniFile))
            # get variable section if not specified header
            if not header:
                header = self.loadVariables[varname].section
            # write value to ini file
            config = cl_template.iniParser(iniFile)
            val = self.serialize(self.getInfo(varname).type,
                  val)
            writeVal = convertStrListDict(val)
            if writeVal != config.getVar(header,varname):
                if not config.setVar(header,{varname: writeVal}):
                    raise DataVarsError(
                        _("Unable to write variable {varname} into "
                          "'{location}'").format(varname=varname,
                                                        location=location))
            return True
        else:
            raise DataVarsError(_("Unable to find alias '%s' for the "
                                  "template variables path")%location)
        return False

    def Delete(self, varname, location='system', header=False):
        """
        Delete variable 'varname' from ini file and refresh value.
        'location' must be ('system', 'local', 'remote') use for specify
        type of ini file location ini. 'header' use for specify section in
        ini file. If header not specified then use default for this value
        section.
        """
        # get path to target ini file
        iniFile = self.getEnvFileByLocation(location)
        if iniFile:
            # make parent directories
            makePath(path.dirname(iniFile))
            # get variable section if not specified header
            if not header:
                if not varname in self.loadVariables:
                    self.loadVariable(varname)
                header = self.loadVariables[varname].section
            # write value to ini file
            config = cl_template.iniParser(iniFile)
            allVars = config.getAreaVars(header)
            if allVars is False:
                return False
            if varname in allVars.keys():
                retDelVar = config.delVar(header,varname)
                # if success delete value and this section empty
                # of ini file then remove it
                return retDelVar and (config.getAreaVars(header) or
                                      config.delArea(header))
            return True
        else:
            raise DataVarsError(_("Unable to find alias '%s' for the "
                                  "template variables path")%location)
        return False

    def Select(self,selField,where="os_disk_dev",eq=None,ne=None,
               _in=None,_notin=None,like=None,notlike=None,func=None,
               sort=None, sortkey=None, limit=None,zipVars=None):
        """Select value from table variables"""
        if zipVars is None:
            zipVars = self.ZipVars
        if func:
            if func.func_code.co_argcount > 1:
                filterFunc = lambda x:func(*x[:func.func_code.co_argcount])
            else:
                filterFunc = func
        elif eq != None:
            filterFunc = lambda x:x[0] == eq
        elif ne != None:
            filterFunc = lambda x:x[0] != ne
        elif _in != None:
            filterFunc = lambda x:x[0] in _in
        elif _notin != None:
            filterFunc = lambda x:not x[0] in _notin
        elif like != None:
            filterFunc = lambda x:re.compile(like).search(x[0])
        elif notlike != None:
            filterFunc = lambda x:not re.compile(notlike).search(x[0])
        else:
            filterFunc = lambda x:x

        if not type(where) in (tuple,list):
            where = [where]
        woffset = len(where)
        if type(selField) in (tuple,list):
            count = len(selField)+woffset
            mapFunc = lambda x:x[woffset:]
            res = filter(filterFunc,
                  zipVars(*(where+selField)))
        else:
            count = 1+woffset
            fields = where+[selField]
            mapFunc = lambda x:x[woffset]
            res = filter(filterFunc,
                  zipVars(*fields))
        if sort:
            if "/" in sort:
                sort,sortkey = sort.split('/')
                sortkey = itemgetter(*map(int,sortkey.split(',')))
            res.sort(key=sortkey,reverse=True if sort == "DESC" else False)
        if limit == 1:
            if not any(res):
                res = [[""]*(count)]
            return mapFunc(res[0])
        else:
            return map(mapFunc,res[:limit])

    def processRefresh(self):
        for refreshVar in self.refresh:
            if not refreshVar.invalid:
                refreshVar.refresh()

    def ZipVars(self,*argvVarNames):
        """
        Get zipped values of variables specified by list 'argvVarNames'
        """
        return zip(*map(self.Get,argvVarNames))

    def close(self):
        for varObj in self.loadVariables.values():
            if hasattr(varObj,"close"):
                varObj.close()
            elif hasattr(varObj,"value") and hasattr(varObj.value,"close"):
                varObj.value.close()

    def addGroup(self,groupName,normal=(),expert=(),brief=(),next_label=None,
                 brief_force=(), hide=(), expert_label=None,image='',
                 custom_buttons=()):
        """
        Добавить группу переменных для метода

        groupName - название группы
        normal - регулярные настройки
        expert - расширенные настройки
        hide -   кортеж имен переменных, которые необходимо скрыть в brief
        image -  изображение
        brief -  кортеж имен переменных, значение которых необходимо отобразите
                 в выводе-подтверждении
        brief_force - кортеж имен переменных, значение которых необходимо
                 отобразите в выводе-подтверждении, даже если параметр является
                 несовместимым
        expert_label - текст вместо стандартной надписи для отображения
                 расширенных параметров
        next_label - название кнопки для запуска действия или перехода к
                 следующей группе
        custom_buttons - список кортежей (название кнопки, текст на кнопке,
                                          значение (метод), элемент ("button")
        """
        default_next_label = _("Next")
        default_expert_label = _("Click for advanced settings")
        self.groups.append(
            {'name':groupName,
             'normal':normal,
             'expert':expert,
             'hide':hide,
             'image':image,
             'brief':brief,
             'custom_buttons':custom_buttons,
             'brief_force':brief_force,
             'expert_label':expert_label or default_expert_label,
             'next_label':next_label or default_next_label})
        lastIndex = len(self.groups)-1
        for var in chain(normal,expert):
            self.mapVarGroup[var] = lastIndex
        return self

    def getGroups(self):
        """
        Get groups variables
        """
        return self.groups

    def clearGroups(self):
        """
        Clear group (using for drop translate)
        """
        self.groups = []
        self.mapVarGroup = OrderedDict()

    def addBrief(self,next_label=None,text=None,image=""):
        self.briefData["next"] = next_label
        self.briefData["help"] = text
        self.briefData["image"] = image

    def getBrief(self):
        return self.briefData

    def _dependSort(self,inlist):
        """
        Sort variable use "check_after" attribute
        """
        moved = []
        inlist = list(inlist)
        while(inlist):
            for i,el in enumerate(islice(inlist,1,None)):
                if el.name in inlist[0].check_after:
                    inlist.insert(0,inlist.pop(i+1))
                    if inlist[0].name in moved:
                        raise VariableError("Loop depends for %s on check"%
                            (",".join(moved)))
                    moved.append(inlist[0].name)
                    break
            else:
                yield inlist.pop(0)
                if inlist:
                    moved = [inlist[0].name]

    def _nocheckSort(self,inlist):
        """
        Place check variable without check (standard check method) at first
        """
        afterlist = []
        for i in inlist:
            if not i.source and \
                not i.check_after and \
                Variable.check.__func__ == i.check.__func__ and \
                Variable.uncompatible.__func__ == i.uncompatible.__func__:
                yield i
            else:
                afterlist.append(i)
        for i in afterlist:
            yield i

    def fixWsdlTableValue(self,varObj,val):
        """
        Fix data get by wsdl (list-list) type
        """
        if varObj.type == "table":
            multiLists = filter(lambda x:"list-list" in x[1],
                         enumerate(
                         [self.getInfo(x).type for x in varObj.source]))
            if multiLists:
                val = list(izip_longest(*val,fillvalue=""))
                for col,typecol in multiLists:
                    if col < len(val):
                        val[col] = map(lambda x:x.split(',') 
                                       if type(x) in StringTypes else x,
                                   val[col])
                val = list(izip_longest(*val,fillvalue=""))
        return val

    def plainList(self,*elements):
        """
        Make plain list from many dimensions
        """
        for w in elements:
            if type(w.message) in (list,tuple):
                for i in self.plainList(*w.message):
                    yield i
            else:
                yield w

    def printGroup(self,info):
        """
        Вывести группу переменных
        """
        print "=DEBUG.SET="
        for vname in sorted(filter(lambda x:x.lower() == x,
                          info._type_info.keys())):
            print "|", vname,"=",getattr(info,vname)
        print "=DEBUG.DATAVARS="
        for vname in self.mapVarGroup.keys():
            print "|", vname, "=", self.Get(vname)
        print "=DEBUG.END="

    def checkGroups(self,info,allvars=False):
        """
        Check variables in group or all in info
        """
        def compare_choice(part_s, *options):
            lower_unicode = lambda x: _toUNICODE(x).lower()
            part_s = lower_unicode(part_s)
            options = [lower_unicode(x) for x in options]
            return any(x == part_s for x in options)

        if not self.groups:
            return []
        errors = []
        if self.groups:
            keys = self.mapVarGroup.keys()
        else:
            keys = sorted(filter(lambda x:x.lower() == x,
                          info._type_info.keys()))
        varsByKeys = map(self.getInfo, keys)
        groupVars = []

        # invalidate default variables
        if hasattr(info,"Default") and info.Default:
            default = info.Default
            for varname in default:
                self.Invalidate(varname,onlySet=True)
        else:
            default = []
        # look over all vars
        for var,varObj in map(lambda x:(x.name,x),
                          self._nocheckSort(self._dependSort(varsByKeys))):
            # if info skipped and send None
            if info is None:
                val = None
            else:
                # if variable not exists in info data - skip it
                if not hasattr(info,var):
                    continue
                # get value of variable from info
                val = getattr(info,var)
            varSendFromClient = not val is None and not var in default
            varGroupUntrusted = varObj.untrusted and var in groupVars
            varAllUntrusted =  varObj.untrusted and allvars
            # if group variable is not defined and
            # variable send from client
            # datavars has groups of variables
            if not groupVars and varSendFromClient and self.groups:
                # get normal and expert variables from group
                groupIndex = self.mapVarGroup[var]
                groupVars = list(self.groups[groupIndex]["normal"]) + \
                            list(self.groups[groupIndex]["expert"])
            if varSendFromClient or varGroupUntrusted or varAllUntrusted \
                or not info:
                try:
                    uncomperr = self.Uncompatible(var)
                    # get value for final or group check
                    if val is None:
                        if not uncomperr:
                            self.Check(var,self.Get(var))
                    else:
                        val = self.fixWsdlTableValue(varObj,val)
                        if varObj.type in ("choice","choice-list"):
                            choicedata = self.ChoiceAndComments(var)
                            if any(choicedata):
                                if choicedata[1] is None:
                                    choicedata = [choicedata[0],
                                                  choicedata[0]]
                                choicedata = zip(*choicedata)
                                if varObj.type == "choice":
                                    vals = [val]
                                else:
                                    vals = val
                                res = []
                                for val in vals:
                                    if not val in map(lambda x:x[0],choicedata):
                                        result = filter(lambda x:compare_choice(val, x[0], x[1]),
                                                 choicedata)
                                        if len(result) > 1:
                                            raise VariableError(_("Ambiguous choice:%s")%
                                                formatListOr(map(
                                                lambda x:'"%s"(%s)'%(x[1],x[0]) if x[0] != x[1] else x[0],
                                                             result)))
                                        elif result:
                                            val = result[0][0]
                                    res.append(val)
                                if varObj.type == "choice":
                                    val = res[0]
                                else:
                                    val = res
                        self.Set(var,val)
                        # raise error for atempt set uncompatible variable
                        if uncomperr:
                            raise VariableError(uncomperr)
                except (VariableError,DataVarsError) as e:
                    # assemble all variable errors
                    messages = e.message if type(e.message) == list \
                               else [e]
                    mess = "\n".join(map(lambda x:str(x),self.plainList(*messages)))
                    mapError = {PasswordError:'pwderror',
                                CommonVariableError:'commonerror'}
                    for k,v in mapError.items():
                        if isinstance(e,k) or \
                            type(e.message) == list and \
                            all(isinstance(x,k) for x in e.message):
                            typeError = v
                            break
                    else:
                        typeError = 'error'
                    errors.append({'type':typeError, 'field':var,
                                   'field_obj': self.getInfo(var) if var else "",
                                   'message':mess})
                except BaseException as e:
                    for i in apply(traceback.format_exception, sys.exc_info()):
                        print i,
                    errors.append({'type':'error', 'field':var,
                                   'message':str(e)})
        return errors

    def printDependence(self):
        for key,val in self.loadVariables.items():
            if val.reqVars:
                print key,"->",[x.name for x in val.reqVars]

    def printVars(self,filt=lambda x:x):
        for i in sorted(filter(filt,self.allVars.keys())):
            print "{0:<40} {1}".format(i,self.Get(i,True))

    def reinit(self):
        """
        Using for update variable translatable elements (example: label)
        """
        for key,var in self.loadVariables.items():
            var.init()

    def printWrong(self,filt=lambda x:x):
        print "!!!WRONG VARS!!!"
        for i in sorted(filter(filt,self.allVars.keys())):
            if self.getInfo(i).untrusted:
                print "{0:<40} {1}".format(i,self.Get(i,True))