calculate-utils-3-lib/calculate/lib/datavars.py

#-*- coding: utf-8 -*-

# Copyright 2008-2012 Calculate Ltd. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import re
import sys
import importlib
from utils.text import convertStrListDict, _toUNICODE, formatListOr
from utils.files import pathJoin
from os import path
import os
from collections import OrderedDict
import time
from threading import RLock
import pyinotify
from itertools import *
import operator
import traceback

def itemgetter(*args,**kwargs):
    """
    itemgetter with alwaysTuple param, which used for force tuple elements
    """
    alwaysTuple = kwargs.get('alwaysTuple',False)
    if alwaysTuple:
        if len(args)==1:
            return lambda x:(x[args[0]],)
    return operator.itemgetter(*args)

class DataVarsError(Exception):
    """Exception of getting variable values"""

class VariableError(Exception):
    """Exception of sended by Variable"""

import cl_template

def makePath(dirs,mode=755):
    if not path.exists(dirs):
        os.makedirs(dirs,mode)

from calculate.lib.cl_lang import setLocalTranslate
setLocalTranslate('cl_lib3',sys.modules[__name__])

READONLY = "r"
WRITEABLE = "w"

class Variable:
    """
    Class of variable. Need for creating calculate utilities variable
    add fill method described by class with fill methods.

    Instance attributes:
      wasSet : boolean
        variable value was set by _Set method
      reqVars : List
        list of variable which use this variable in fill method
      invalid : boolean
        invalid value
      section : string
        section of variable (main,install,builder and etc)
    """
    # label of variable
    label = None
    # opts for cmdline
    opt = None
    # metavalues
    metavalue = None
    # help information
    help = None
    # writeable variable
    mode = WRITEABLE
    # hidden variable
    hide = True
    # default value
    value = None
    # variable section
    section = "main"
    # variable name
    name = ""
    # gui label
    label = None
    # opt for console
    opt = None
    # help for console
    help = None
    # variable type: string, multichoice, list, number, choiceedit, table
    # flag, yesno, password
    type = "string"
    # metavalue for opt
    metavalue = None
    # source column with data
    source = None
    # gui elelemnt type
    element = None
    # force check after variables in list
    check_after = []
    # fill value is true
    untrusted = False
    NONE,CHECK,UNCOMPAT,HUMAN = range(4)

    def __init__(self,parent=None,**kwargs):
        """
        Initialize instance
        """
        self.modeGet = Variable.NONE
        self.invalid = self.value is None
        self.wasSet = False     # variable is user or ini set
        self.reqVars = []       # varaibles which used this value for fill
        self.reqCheck = []      # variables which used this value for check
        self.reqUncompat = []   # varaibles which used for uncompat
        self.variableType = type
        self.parent = parent
        self.name = self.getVariableName()
        self.init()
        for key,val in kwargs.items():
            setattr(self,key,val)

    def init(self):
        """
        Overridible init using for palce translatable strings
        """
        pass

    @classmethod
    def getVariableName(cls):
        return re.sub("(.)([A-Z])","\\1_\\2",cls.__name__[8:]).lower()

    def Get(self,varname=None,humanreadable=False):
        """Get value of other value"""
        if not varname or varname == self.name:
            return self._get()

        varObj = self.parent.getInfo(varname)
        #varObj.modeGet = Variable.UNCOMPAT
        res = self.parent.Get(varname,humanreadable)

        # build dependence for check and uncompatible
        if varObj:
            if self.modeGet == Variable.NONE:
                if not self in varObj.reqVars:
                    varObj.reqVars.append(self)
            if self.modeGet == Variable.CHECK:
                if not self in varObj.reqCheck:
                    varObj.reqCheck.append(self)
            elif self.modeGet == Variable.UNCOMPAT:
                if not varObj in self.reqUncompat:
                    self.reqUncompat.append(varObj)
        return res

    def Choice(self,varname):
        """Choice from other value"""
        return self.parent.Choice(varname)

    def Select(self,selField,**kwargs):
        """Select by datavars"""
        return self.parent.Select(selField,zipVars=self.ZipVars,**kwargs)

    def ZipVars(self,*argvVarNames):
        """
        Get zipped values of variables specified by list 'argvVarNames'
        """
        return zip(*map(self.Get,argvVarNames))

    def get(self):
        """
        Overridable
        Calculate value method
        """
        if self.__class__.value:
            return self.__class__.value
        else:
            return [] if "list" in self.type else ""

    def humanReadable(self):
        """Return human readable value"""
        val = self.Get()
        if "choice" in  self.type:
            choiceVal, commentVal = self._choice()
            if commentVal:
                valDict = dict(zip(choiceVal,commentVal))
                if "list" in self.type:
                    return map(lambda x:valDict.get(x,x),val)
                else:
                    return valDict.get(val,val)
        return val

    def _human(self):
        # TODO: may be rest to NONE
        try:
            oldModeGet = self.modeGet
            self.modeGet = Variable.HUMAN
            return self.humanReadable()
        finally:
            self.modeGet = oldModeGet


    def getHumanReadableAuto(self):
        """
        Return humanreadable values for readonly variables
        """
        if self.mode == READONLY:
            return self._human()
        else:
            return self._get()

    def _get(self):
        """
        Standard inner method for getting value of variable.
        """
        # variable need refill
        if self.invalid:
            # get net value
            try:
                oldModeGet = self.modeGet
                self.modeGet = Variable.NONE
                value = self.get()
            finally:
                self.modeGet = oldModeGet
            # if value not equal previous value
            if value != self.value:
                # invalidate depended variables
                self.invalidate()
            # mark as filled
            self.invalid = False
            if hasattr(self.value,"close"):
                self.value.close()
            self.value = value
        return self.value

    def choice(self):
        """
        Overridable

        Generation list of allowable values
        """
        return []

    def _choice(self):
        """
        Convert choice created by variable choice to two tuple (values,comments)
        """
        res = self.choice()
        if res and type(res[0]) in (tuple,list):
            return zip(*res)
        else:
            return (res,None)

    def check(self,value):
        """
        Overridable

        Check method, for wrong value need raise VariableError
        """
        pass

    def uncompatible(self):
        """
        Overridable

        Using for check uncompatibility using variable with value of
        other variables.
        """
        return ""

    def _uncompatible(self):
        """
        Full check uncompatible
        """
        try:
            self.modeGet = Variable.UNCOMPAT
            self.reqUncompat = []
            return self.uncompatible()
        finally:
            self.modeGet = Variable.NONE

    def set(self,value):
        """
        Overridable

        Using for replace value before set
        """
        def convert(value):
            if value and value.lower() != "auto":
                return "on" if self.isTrue(value) else "off"
            else:
                return ""
        if self.type == "bool":
            value = "on" if self.isTrue(value) else "off"
        if "bool" in self.type and "list" in self.type:
            return map(convert,value)
        return value

    def checkType(self,value):
        """Check value for type"""
        if "list" in self.type:
            if not type(value) in (list,tuple):
                raise VariableError(
                    _("Value for {varname} may be '{vartype}' only").format(
                    varname=self.label or self.name,
                    vartype="list"))
            error = _("Values for {varname} may be {vartype} only")
        else:
            value = repeat(value,1)
            error = _("Value for {varname} may be {vartype} only")
        if "string" in self.type:
            value, valuecopy = tee(value,2)
            for v in (x for x in valuecopy if not type(x) in (str,unicode)):
                raise VariableError(error.format(
                    varname=self.label or self.name,
                    vartype="string"))
            return
        if "choice" in self.type:
            choiceVal = self.choice()
            if choiceVal and type(choiceVal[0]) in (tuple,list):
                choiceVal = [x[0] for x in choiceVal]
        for val in value:
            if "choice" in self.type and not "choiceedit" in self.type:
                if choiceVal and val and not val in choiceVal or val is None:
                    name = self.label or self.name
                    raise VariableError(error.format(
                        varname=name,
                        vartype=formatListOr(choiceVal)))

    def setValue(self,value):
        """
        Standard action for set value
        """
        self.value = self.set(value)
        self.wasSet  = True
        self.invalid = False
        # run check
        self._check()

    def _set(self,value,force=False):
        """
        Standard inner method for setting value for variable.
        """
        # runc check choice
        try:
            self.modeGet = Variable.CHECK
            self.checkType(value)
        finally:
            self.modeGet = Variable.NONE
        # if value change
        if value != self.value or type(value) in (list,tuple):
            self.invalidate(True)
        if hasattr(self.value,"close"):
            self.value.close()
        # run set value
        self.setValue(value)

    def _check(self,value=None):
        """
        Full check value
        """
        if value is None:
            value = self.Get(self.name)
        try:
            self.modeGet = Variable.CHECK
            self.untrusted = True
            self.check(value)
            self.untrusted = False
        finally:
            self.modeGet = Variable.NONE

    def invalidate(self,force=False):
        """
        Invalidate value of variable (drop filled value).

        force : boolean=False
          Drop value also for after manual set
        """
        if force or not self.wasSet:
            self.wasSet = False
            self.invalid = True
            for var in self.reqVars:
                var.invalidate()
            for var in self.reqCheck:
                var.untrusted = True
            self.reqVars = []
            self.reqCheck = []

    @classmethod
    def isTrue(self,value):
        if type(value) == bool:
            return value
        if  value.lower() in ('yes','on','true'):
            return True
        return False

class TableVariable(Variable):
    """
    Table type variable
    """
    type = "table"

    def get(self,hr=False):
        """Get table data"""
        for varname,value in ifilter(lambda x:type(x[1]) != list,
                             imap(lambda x:(x,self.Get(x)),
                             self.source)):
            raise VariableError(
                _("Source variable %s not contains list")%varname)
        return map(list,
               izip_longest(
               *map(lambda x:self.Get(x,humanreadable=hr),
               self.source),fillvalue='')) or [[]]

    def getHumanReadableAuto(self):
        """
        Return humanreadable values for readonly variables
        """
        return self.get(hr=None)

    def humanReadable(self):
        return self.get(hr=True)

    def __getWritableColumns(self,includeFirst=False):
        """
        Get data for writable columns exclude index column
        (Example: (1,'os_disk_mount)
        """
        if includeFirst:
            offset = 0
        else:
            offset = 1
        return filter(lambda x:self.parent.getInfo(x[1]).mode == WRITEABLE,
               enumerate(self.source[offset:]))

    def check(self,value):
        """
        Check table value - check all writeable columns.
        """
        writeCols = self.__getWritableColumns(includeFirst=True)
        error = []
        if not any(value):
            value = [[]]*len(writeCols)
        else:
            value = zip(*map(itemgetter(*map(itemgetter(0),
                                writeCols),alwaysTuple=True),value))
        for colInfo, values in \
            zip(writeCols,value):
            try:
                self.parent.Check(colInfo[1],values)
            except VariableError as e:
                error.append(e)
        if error:
            raise VariableError(error)

    def __isIndexWritable(self):
        return self.parent.getInfo(self.source[0]).mode == WRITEABLE

    def checkType(self,value):
        """Check table value type"""
        # check type value (list and each element is list)
        if not type(value) in (list,tuple) or \
            any(i for i in value if not type(i) in (tuple,list)):
            raise VariableError(
                    _("Value for {varname} may be '{vartype}' only").format(
                    varname=self.label or self.name,
                    vartype="table"))
        # check len all entries
        writeLen = len(self.__getWritableColumns())+1
        for item in value:
            if item and not len(item) in (writeLen,len(self.source)):
                raise DataVarsError(
                    _("Wrong entry '{entry}' for table variable '{varname}'").
                    format(entry=str(item),
                    varname=(self.label or self.name).lower()))

        # check rewrite readonly index
        if not self.__isIndexWritable():
            indexValues = self.Get(self.source[0])
            for item in value:
                if item and not item[0] in indexValues:
                    raise DataVarsError(
                        _("Attempt to rewrite a readonly index field "
                        "{fieldname} in variable {variablename}").format(
                        fieldname=self.source[0],variablename=self.name))

    def setValue(self,value):
        self.untrusted = True
        oldvalue = self.Get(self.name)
        # get writable columns
        writeCols = self.__getWritableColumns()
        # get slicer
        _slice = itemgetter(*map(itemgetter(0),writeCols),
                            alwaysTuple=True)
        # create dict for writable columns
        # if table not empty
        if any(oldvalue):
            oldvalue = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
                                   oldvalue))
        else:
            oldvalue = OrderedDict()
        # get new dict
        if any(value):
            if len(value[0]) == len(self.source):
                newval = OrderedDict(map(lambda x:(x[0],_slice(x[1:])),
                                     value))
            else:
                newval = OrderedDict(map(lambda x:(x[0],x[1:]),value))
        else:
            newval = OrderedDict()
        # if table with writable index field then replace all table
        error = []
        if self.__isIndexWritable():
            if any(value):
                self.parent.Check(self.source[0],zip(*value)[0])
            oldvalue = newval
            try:
                self.parent.Set(self.source[0],oldvalue.keys())
            except VariableError,e:
                error.append(e)
        # update entry by index field
        else:
            oldvalue.update(newval)
        oldvalValues = zip(*oldvalue.values())
        for col,vals in zip(map(lambda x:x[1],writeCols),
                            oldvalValues):
            try:
                self.parent.Set(col,list(vals))
            except VariableError,e:
                error.append(e)
        for num,varname in writeCols[len(oldvalValues):]:
            self.parent.Invalidate(varname,True)
        if error:
            raise VariableError(error)
        else:
            self.untrusted = False

class ReadonlyVariable(Variable):
    """
    Alias for readonly variables
    """
    mode = READONLY

class FieldValue:
    """
    Table column variable
    """
    type = "list"
    source_variable = ""
    column = 0

    def get(self):
        sourceVar = self.Get(self.source_variable)
        if any(sourceVar):
            return zip(*sourceVar)[self.column]
        else:
            return []

class SourceReadonlyVariable(ReadonlyVariable):
    """
    Source readonly variable (Using for table)
    Example:

    ['os_location_source',...,'os_location_size'(this)]
    """
    indexField = ""

    def get(self):
        return map(lambda x:x or "",
               map(self.getMap().get,
               self.Get(self.indexField)))

    def humanReadable(self):
        return map(lambda x:x or "",
               map(self.getMapHumanReadable().get,
               self.Get(self.indexField)))

    def getMap(self):
        return {}

    def getMapHumanReadable(self):
        return self.getMap()

class ReadonlyTableVariable(TableVariable):
    """
    Alias for readonly table
    """
    mode = READONLY

class FileVariable(Variable):
    """
    Test variable
    """
    filename = ""
    directory = ""
    recursive = False
    wm = None

    def init(self):
        if self.wm is None:
            self.wm = pyinotify.WatchManager()
            if self.filename:
                self.mask = pyinotify.IN_MODIFY
            else:
                self.mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE
            self.notifier = pyinotify.Notifier(self.wm,timeout=1)
            self.wdd = self.wm.add_watch(self.filename or self.directory,
                                         self.mask,
                                         rec=self.recursive)

    def close(self):
        self.notifier.stop()

    def refresh(self):
        if self.notifier.check_events():
            self.notifier.read_events()
            self.parent.Invalidate(self.name)

class SimpleDataVars:
    """
    Simple datavars for temporary link var objs.
    """
    def __init__(self,*objs):
        self.allVars = {}
        self.cache = {}
        for obj in objs:
            obj.parent = self
            self.allVars[obj.getVariableName()] = obj

    def Get(self,varname,humanreadable=False):
        if not varname in self.cache:
            self.cache[varname] = self.allVars.get(varname).get()
        return self.cache[varname]

    def getInfo(self,varname):
        return self.allVars.get(varname, None)

    def flIniFileFrom(self,iniFile):
        """
        Read variable values from ini files.
        Return False if ini file is invalid.
        """
        config = cl_template.iniParser(iniFile)
        for varname in self.allVars.keys():
            if not varname in self.cache:
                val = config.getVar('main',varname)
                if val:
                    self.cache[varname] = val.encode('utf-8')

class DataVars:
    """
    Class contains variables, perform creating variable objects by
    class variable describer and variable filler.

    Instance attributes:
      moduleList : Dict
        dictionary has section name with modules which describe variable
        for this sectin
      loadVariables : Dict
        contains all initialized variables load from varObj and fillObj
      requestVariables : List
        contains var object which already run Get method. Use for detect
        dependence of variables.
    """
    defaultData = 'calculate.lib.variables'
    l = RLock()

    def __init__(self):
        self.requestVariables = []
        self.loadVariables = {}
        self.allVars = {}
        self.untrustedVariables = set()
        self.refresh = []
        self.groups = []
        self.mapVarGroup = OrderedDict()
        self.briefData = {}

    def importData(self,data=None):
        """
        Import modules for variables. 'section' is section name, 'modList'
        contains tuple with modules which discribe variable names and fill
        methods
        """
        # try import vars and fill modules specified by modList
        if not data:
            data = self.defaultData
        try:
            varModule = importlib.import_module(data)
        except (ImportError,AttributeError),e:
            raise DataVarsError("\n".join([
                _("Failed to import module %s")%data,
                _("error")  + ": " +str(e)]))
        # get all Variable classes from module
        # variable of class VariableClAction will be named cl_action
        if hasattr(varModule,"section"):
            section = varModule.section
        else:
            section = "main"
        for varMod in map(lambda x:getattr(varModule,x),
                      filter(lambda x:not x.startswith("_") and \
                             not "Variable" in x,
                      dir(varModule)))+[varModule]:
            for classname in \
                    ifilterfalse(("ReadonlyVariable",
                                 "Variable","VariableError").__contains__,
                    ifilter(lambda x:x.startswith("Variable"),
                    dir(varMod))):
                varObj = getattr(varMod,classname)
                varName = varObj.getVariableName()
                self.allVars[varName] = (section,varObj)

    def stubVariable(self,varname,*args,**kwargs):
        raise DataVarsError(_("Variable %s not found")%varname)

    def loadVariable(self,varname):
        """
        Initialize variable 'varname' from fillObj and varObj.
        """
        if not self.allVars:
            self.importData()
        if not varname in self.loadVariables:
            if varname in self.allVars.keys():
                section,varObj = self.allVars[varname]
                newVar = varObj(parent=self,section=section)
                if hasattr(newVar,"refresh"):
                    self.refresh.append(newVar)
                #attrs = (("Fill","get_%s"),
                #         ("Check","check_%s"),
                #         ("Choice","choice_%s"),
                #         ("Uncompatible","uncompatible_%s"),
                #         ("HumanRead","human_%s"),
                #         ("Replace","set_%s"))
                #for attrname,funcname in attrs:
                #    funcname = funcname%varname
                #    if hasattr(varObj,funcname):
                #        setattr(newVar,attrname,getattr(varObj,funcname))
                #setattr(newVar,"Get",self.Get)
                self.loadVariables[varname] = newVar
            else:
                self.loadVariables[varname] = \
                    self.stubVariable(varname,parent=self,section="main")

    def defined(self,varName):
        """
        Return exists or not varName
        """
        return varName in self.allVars.keys()

    def getIniVar(self,fullVarName):
        """
        Get variable values from ini files. Variable specified by
        'fullVarName' in format: section 'dot' variable. If specified
        only variable then section will be 'main'

        Example:
        self.getIniVar('install.os_install_net_settings')
        self.getIniVar('cl_action')
        """
        calculateIniFiles = self.Get('cl_env_path')
        section, dot, varname = fullVarName.rpartition('.')
        if not section:
            section = "main"
        retVal = ""
        for iniFile in calculateIniFiles:
            if path.exists(iniFile):
                config = cl_template.iniParser(iniFile)
                data = config.getVar(section,varname,checkExistVar=True)
                if data is False:
                    return False
                existsVar, value = data
                if existsVar:
                    retVal = value
        return retVal.encode('UTF-8')

    def getRemoteInfo(self, envFile):
        """
        Get information variables from envFile
        Return dict:
        section : { variable1 : value,
                    variable2 : value }
        section2 : { variable : value }
        """
        optionsInfo = {}
        config = cl_template.iniParser(envFile)
        allsect = config.getAllSectionNames()
        if allsect:
            for section in allsect:
                allvars = config.getAreaVars(section)
                if allvars == False:
                    return False
                # options (dict of variables:value)
                options = {}
                for varName, value in allvars.items():
                    varName = varName.encode("UTF-8")
                    value=convertStrListDict(value.encode("UTF-8"))
                    options[varName] = value
                # if options exists add section to return dict as key section
                if options:
                    optionsInfo[section] = options
        return optionsInfo

    def flIniFile(self,raiseOnError=False):
        """
        Read variable values from ini files.
        Return False if ini file is invalid.
        """
        calculateIniFiles = self.Get('cl_env_path')
        # get initialized section names
        sections = set(map(lambda x:x[0],self.allVars.values()))
        res = True
        for iniFile in calculateIniFiles:
            if path.exists(iniFile):
                config = cl_template.iniParser(iniFile)
                iniSections = config.getAllSectionNames()
                if not iniSections:
                    continue
                iniSections = tuple(set(iniSections)&sections)
                for section in iniSections:
                    importVars = config.getAreaVars(section)
                    if self.allVars == False:
                        return False
                    for key,value in importVars.items():
                        try:
                            key = _toUNICODE(key)
                            origkey = key.encode('utf-8')
                            value = convertStrListDict(_toUNICODE(value))
                            if not key in self.loadVariables:
                                self.loadVariable(origkey)#,onlySection=section)
                            varObj = self.loadVariables[origkey]
                            if "list" in varObj.type and \
                                not type(value) in (list,tuple) :
                                value = map(lambda x:x.strip().encode('utf-8'),
                                        value.split(','))
                            self.Set(origkey,value)
                        except Exception as e:
                            res = False
                            if raiseOnError:
                                raise
                            else:
                                sys.stdout.write(_("Error on read ini file: "))
                                sys.stdout.write(str(e)+"\n")
                                sys.stdout.flush()
        return res

    def Get(self,varname,humanreadable=False):
        """Threading safety Get"""
        DataVars.l.acquire()
        try:
            var = self.__Get(varname,humanreadable)
        finally:
            pass
            DataVars.l.release()
        return var

    def GetBool(self,varname):
        return Variable.isTrue(self.Get(varname))

    def __Get(self, varname, humanreadable=False):
        """
        Get value of variable 'varname'
        """
        try:
            #tm = time.time()
            # load variable if it isn't loaded
            if not varname in self.loadVariables:
                self.loadVariable(varname)
            varObj = self.loadVariables[varname]
            # if for this variable already use Get method
            if varObj.invalid and varObj in self.requestVariables:
                varnames = "-".join(map(lambda x:x.name,
                                    self.requestVariables+[varObj]))
                raise DataVarsError(
                    _("Loop dependence of variables '%s'")%varnames)
            # add this variable for list of requested variables
            if not humanreadable:
                self.requestVariables.append(varObj)
            if humanreadable is None:
                res = varObj.getHumanReadableAuto()
            elif humanreadable is True:
                res = varObj._human()
            else:
                res = varObj._get()
            if not humanreadable:
                self.requestVariables.pop()
            return res
        except BaseException,e:
            while(len(self.requestVariables)):
                self.requestVariables.pop()
            raise

    def Check(self,varname,value):
        """
        Check varialbe value
        """
        # load variable if it isn't loaded
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        varObj.checkType(value)
        varObj._check(value)

    def Choice(self, varname):
        """
        Get choice value of variable 'varname'
        """
        # load variable if it isn't loaded
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        return varObj._choice()[0]

    def Uncompatible(self, varname):
        """
        Check on compatible change value of this variable

        Empty string if variable changable or string with error
        """
        # load variable if it isn't loaded
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        return varObj._uncompatible()

    def getRequired(self,varname):
        """
        Generate list variables using in uncompatible check
        """
        added = []
        def genReqs(varObj):
            if not varObj in added:
                yield varObj
                for var in varObj.reqVars:
                    for dep in genReqs(var):
                        yield dep
                if hasattr(varObj,"source") and varObj.source:
                    for varsource in imap(self.getInfo,varObj.source):
                        for dep in genReqs(varsource):
                            yield dep
        for dep in genReqs(self.getInfo(varname)):
            added.append(dep)
        return added

    def ChoiceAndComments(self,varname):
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        return varObj._choice()

    def getInfo(self, varname):
        """
        Get info (label and other for variable)
        """
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        return self.loadVariables[varname]

    def Set(self, varname, varvalue,force=False):
        """
        Set value 'varvalue' for variable 'varname'
        """
        varObj = self.getInfo(varname)
        if force or varObj.mode != READONLY:
            varObj._set(varvalue)
        else:
            raise DataVarsError(
                _("Attempt to rewrite a readonly variable %s")%
                varname)

    def Invalidate(self,varname,force=False,onlySet=False):
        """
        Invalidate value of variable. Force - invalidate also for set values
        """
        if not varname in self.loadVariables:
            self.loadVariable(varname)
        varObj = self.loadVariables[varname]
        if not onlySet or varObj.wasSet:
            varObj.invalidate(force or onlySet)

    def getEnvFileByLocation(self,location):
        """
        Get env file path by location alias. This path append to value
        of variable 'cl_chroot_path'.
        Return False if alias not found.
        """
        calculateIniFile = self.Select('cl_env_path',
                                       where='cl_env_location',
                                       eq=location,limit=1)
        if calculateIniFile:
            return pathJoin(self.Get('cl_chroot_path'),calculateIniFile)
        else:
            return False

    def Write(self, varname, val=None, force=False, location='default',
              header=False):
        """
        Write variable 'varname' to ini file. If set 'val', then to
        variable before write perform value changing. 'force' use for
        change value of read-only variables. 'location' must be ('default',
        'local', 'remote') use for specify type of ini file location ini.
        'header' use for specify section in ini file. If header not
        specified then use default for this value section.
        """
        # set value to variable if specified value
        if not val is None:
            self.Set(varname,val,force=force)
        val = self.Get(varname)
        # get path to target ini file
        iniFile = self.getEnvFileByLocation(location)
        if iniFile:
            # make parent directories
            makePath(path.dirname(iniFile))
            # get variable section if not specified header
            if not header:
                header = self.loadVariables[varname].section
            # write value to ini file
            config = cl_template.iniParser(iniFile)
            if "list" in self.getInfo(varname).type:
                val = ",".join(val)
            if not config.setVar(header,{varname: convertStrListDict(val)}):
                raise DataVarsError(_("Unable to write variable %s")%varname)
            return True
        else:
            raise DataVarsError(_("Unable to find alias '%s' of the path"
                  " for storage of template variables")%location)
        return False

    def Delete(self, varname, location='default', header=False):
        """
        Delete variable 'varname' from ini file and refresh value.
        'location' must be ('default', 'local', 'remote') use for specify
        type of ini file location ini. 'header' use for specify section in
        ini file. If header not specified then use default for this value
        section.
        """
        # get path to target ini file
        iniFile = self.getEnvFileByLocation(location)
        if iniFile:
            # make parent directories
            makePath(path.dirname(iniFile))
            # get variable section if not specified header
            if not header:
                header = self.loadVariables[varname].section
            # write value to ini file
            config = cl_template.iniParser(iniFile)
            allVars = config.getAreaVars(header)
            if allVars is False:
                return False
            if varname in allVars.keys():
                retDelVar = config.delVar(header,varname)
                # if success delete value and this section empty
                # of ini file then remove it
                return retDelVar and (config.getAreaVars(header) or
                                      config.delArea(header))
            return True
        else:
            raise DataVarsError(_("Unable to find alias '%s' of the path"
                  " for storage of template variables")%location)
        return False

    def Select(self,selField,where="os_disk_dev",eq=None,ne=None,
               _in=None,like=None,notlike=None,func=None,
               sort=None, sortkey=None, limit=None,zipVars=None):
        """Select value from table variables"""
        if zipVars is None:
            zipVars = self.ZipVars
        if func:
            filterFunc = func
        elif eq != None:
            filterFunc = lambda x:x[0] == eq
        elif ne != None:
            filterFunc = lambda x:x[0] != ne
        elif _in != None:
            filterFunc = lambda x:x[0] in _in
        elif like != None:
            filterFunc = lambda x:re.compile(like).search(x[0])
        elif notlike != None:
            filterFunc = lambda x:not re.compile(notlike).search(x[0])
        else:
            filterFunc = lambda x:x

        if type(selField) in (tuple,list):
            count = len(selField)
            mapFunc = lambda x:x[1:]
            res = filter(filterFunc,
                  zipVars(where,*selField))
        else:
            count = 1
            fields = [where,selField]
            mapFunc = lambda x:x[1]
            res = filter(filterFunc,
                  zipVars(where,selField))
        if sort:
            if "/" in sort:
                sort,sortkey = sort.split('/')
                sortkey = itemgetter(*map(int,sortkey.split(',')))
            res.sort(key=sortkey,reverse=True if sort == "DESC" else False)
        if limit == 1:
            if not any(res):
                res = [[""]*(count+1)]
            return mapFunc(res[0])
        else:
            return map(mapFunc,res[:limit])

    def processRefresh(self):
        for refreshVar in self.refresh:
            if not refreshVar.invalid:
                refreshVar.refresh()

    def ZipVars(self,*argvVarNames):
        """
        Get zipped values of variables specified by list 'argvVarNames'
        """
        return zip(*map(self.Get,argvVarNames))

    def close(self):
        for varObj in self.loadVariables.values():
            if hasattr(varObj,"close"):
                varObj.close()
            elif hasattr(varObj,"value") and hasattr(varObj.value,"close"):
                varObj.value.close()

    def addGroup(self,groupName,normal=(),expert=(),brief=(),next_label=None,
                 hide=(), expert_label=None,image=''):
        """
        Add group by name, normal, expert and next_label
        """
        default_next_label = _("Next")
        default_expert_label = _("Click for advanced settings")
        self.groups.append(
            {'name':groupName,
             'normal':normal,
             'expert':expert,
             'hide':hide,
             'image':image,
             'brief':brief,
             'expert_label':expert_label or default_expert_label,
             'next_label':next_label or default_next_label})
        lastIndex = len(self.groups)-1
        for var in chain(normal,expert):
            self.mapVarGroup[var] = lastIndex

    def getGroups(self):
        """
        Get groups variables
        """
        return self.groups

    def clearGroups(self):
        """
        Clear group (using for drop translate)
        """
        self.groups = []
        self.mapVarGroup = OrderedDict()

    def addBrief(self,next_label=None,text=None,image=""):
        self.briefData["next"] = next_label
        self.briefData["help"] = text
        self.briefData["image"] = image

    def getBrief(self):
        return self.briefData

    def _dependSort(self,inlist):
        """
        Sort variable use "check_after" attribute
        """
        moved = []
        inlist = list(inlist)
        while(inlist):
            for i,el in enumerate(islice(inlist,1,None)):
                if el.name in inlist[0].check_after:
                    inlist.insert(0,inlist.pop(i+1))
                    if inlist[0].name in moved:
                        raise VariableError("Loop depends for %s on check"%
                            (",".join(moved)))
                    moved.append(inlist[0].name)
                    break
            else:
                yield inlist.pop(0)
                if inlist:
                    moved = [inlist[0].name]

    def _nocheckSort(self,inlist):
        """
        Place check variable without check (standard check method) at first
        """
        afterlist = []
        for i in inlist:
            if not i.source and \
                Variable.check.__func__ == i.check.__func__ and \
                Variable.uncompatible.__func__ == i.uncompatible.__func__:
                yield i
            else:
                afterlist.append(i)
        for i in afterlist:
            yield i

    def checkGroups(self,info,allvars=False):
        """
        Check variables in group or all in info
        """
        errors = []
        if self.groups:
            keys = self.mapVarGroup.keys()
        else:
            keys = sorted(filter(lambda x:x.lower() == x,
                          info._type_info.keys()))
        varsByKeys = map(self.getInfo, keys)
        groupVars = []

        # invalidate default variables
        if hasattr(info,"Default") and info.Default:
            default = info.Default
            for varname in default:
                self.Invalidate(varname,onlySet=True)
        else:
            default = []
        # look over all vars
        for var,varObj in map(lambda x:(x.name,x),
                          self._nocheckSort(self._dependSort(varsByKeys))):
            # if info skipped and send None
            if info is None:
                val = None
            else:
                # if variable not exists in info data - skip it
                if not hasattr(info,var):
                    continue
                # get value of variable from info
                val = getattr(info,var)
            varSendFromClient = not val is None and not var in default
            varGroupUntrusted = varObj.untrusted and var in groupVars
            varAllUntrusted =  varObj.untrusted and allvars
            # if group variable is not defined and
            # variable send from client
            # datavars has groups of variables
            if not groupVars and varSendFromClient and self.groups:
                # get normal and expert variables from group
                groupIndex = self.mapVarGroup[var]
                groupVars = list(self.groups[groupIndex]["normal"]) + \
                            list(self.groups[groupIndex]["expert"])
            if varSendFromClient or varGroupUntrusted or varAllUntrusted \
                or not info:
                try:
                    uncomperr = self.Uncompatible(var)
                    # get value for final or group check
                    if val is None:
                        if not uncomperr:
                            self.Check(var,self.Get(var))
                    else:
                        self.Set(var,val)
                        # raise error for atempt set uncompatible variable
                        if uncomperr:
                            raise VariableError(uncomperr)
                except VariableError as e:
                    # assemble all variable errors
                    messages = e.message if type(e.message) == list \
                               else [e.message]
                    mess = "\n".join(map(lambda x:str(x),messages))
                    errors.append({'type':'error', 'field':var, 'message':mess})
                except BaseException as e:
                    for i in apply(traceback.format_exception, sys.exc_info()):
                        print i
                    errors.append({'type':'error', 'field':var,
                                   'message':str(e)})
        return errors

    def printDependence(self):
        for key,val in self.loadVariables.items():
            if val.reqVars:
                print key,"->",[x.name for x in val.reqVars]

    def printVars(self,filt=lambda x:x):
        for i in sorted(filter(filt,self.allVars.keys())):
            print "{0:<40} {1}".format(i,self.Get(i,True))

    def reinit(self):
        """
        Using for update variable translatable elements (example: label)
        """
        for key,var in self.loadVariables.items():
            var.init()

    def printWrong(self,filt=lambda x:x):
        print "!!!WRONG VARS!!!"
        for i in sorted(filter(filt,self.allVars.keys())):
            if self.getInfo(i).untrusted:
                print "{0:<40} {1}".format(i,self.Get(i,True))