@ -23,18 +23,19 @@ from calculate.lib.utils.files import (
pathJoin , PercentProgress , getProgPath , process , STDOUT , removeDir ,
pathJoin , PercentProgress , getProgPath , process , STDOUT , removeDir ,
makeDirectory , writeFile , readLinesFile , chmod , chown , FilePermission ,
makeDirectory , writeFile , readLinesFile , chmod , chown , FilePermission ,
find , FindFileType , removeFileWithEmptyDirectory ,
find , FindFileType , removeFileWithEmptyDirectory ,
copyWithPath )
copyWithPath , countFiles )
from calculate . lib . utils . git import Git
from calculate . lib . utils . git import Git
from calculate . lib . utils . portage import Layman , EmergeLog , EmergeLogNamedTask , \
from calculate . lib . utils . portage import Layman , EmergeLog , EmergeLogNamedTask , \
InstalledPackageInfo , EbuildInfoError , EbuildInfo , ChrootEix , \
InstalledPackageInfo , EbuildInfoError , EbuildInfo , ChrootEix , \
get_packages_files_directory , get_manifest_files_directory
get_packages_files_directory , get_manifest_files_directory
from calculate . update . emerge_parser import EmergeParser , \
from calculate . update . emerge_parser import EmergeParser , \
EmergeError , Chroot EmergeCommand, Chroot CommandExecutor, RevdepPercentBlock , \
EmergeError , EmergeCommand, Chroot , Linux32 , CommandExecutor, \
EmergeCache
RevdepPercentBlock, EmergeCache
from calculate . lib . cl_log import log
from calculate . lib . cl_log import log
from . build_storage import Build
from . build_storage import Build
from calculate . update . update import Update
from calculate . update . update import Update
from calculate . install . distr import Distributive
from calculate . install . distr import Distributive
from functools import partial
import os
import os
from os import path
from os import path
from . datavars import BuilderError
from . datavars import BuilderError
@ -44,7 +45,6 @@ from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate, _)
setLocalTranslate ( ' cl_builder3 ' , sys . modules [ __name__ ] )
setLocalTranslate ( ' cl_builder3 ' , sys . modules [ __name__ ] )
__ = getLazyLocalTranslate ( _ )
__ = getLazyLocalTranslate ( _ )
class Builder ( Update ) :
class Builder ( Update ) :
""" Основной объект для выполнения действий связанных с о сборкой системы
""" Основной объект для выполнения действий связанных с о сборкой системы
@ -57,7 +57,7 @@ class Builder(Update):
def mount_target ( self , target ) :
def mount_target ( self , target ) :
dir_distro = target . convertToDirectory ( )
dir_distro = target . convertToDirectory ( )
dir_distro . mountSystemDirectories ( )
dir_distro . mountSystemDirectories ( skip = ( ) )
return True
return True
def umount_system ( self , target ) :
def umount_system ( self , target ) :
@ -115,20 +115,6 @@ class Builder(Update):
image . eventPrepareIso . connect ( self . prepare_iso )
image . eventPrepareIso . connect ( self . prepare_iso )
return True
return True
#def test_builder(self):
# path_chroot = self.clVars.Get('cl_builder_path')
# d1 = dict(zip(
# map(lambda x: pathJoin(path_chroot, x),
# self.clVars.Get('cl_builder_repository_location')),
# self.clVars.Get('cl_builder_repository_name')))
# print d1
# d2 = dict(zip(self.clVars.Get('update.cl_update_rep_path'),
# self.clVars.Get('update.cl_update_rep_name')))
# print d2
# print "!!!!!!",self.clVars.Get('cl_builder_linux_datavars').Get('cl_update_rep_path')
# print "!!!!!!",self.clVars.Get('cl_builder_profile_datavars').Get('cl_update_rep_path')
# return True
def remove_repositories ( self , repname ) :
def remove_repositories ( self , repname ) :
dv = self . clVars . Get ( ' cl_builder_linux_datavars ' )
dv = self . clVars . Get ( ' cl_builder_linux_datavars ' )
rpath = dv . Select ( " cl_update_rep_path " ,
rpath = dv . Select ( " cl_update_rep_path " ,
@ -225,10 +211,9 @@ class Builder(Update):
printERROR = self . printERROR )
printERROR = self . printERROR )
def execute_command ( cmd , lang ) :
def execute_command ( cmd , lang ) :
chroot_cmd = getProgPath ( " /usr/bin/chroot " , )
chroot_path = self . clVars . Get ( ' cl_builder_path ' )
chroot_path = self . clVars . Get ( ' cl_builder_path ' )
return process ( chroot_cmd , chroot_path ,
return self . chroot_process (
cmd , lang = lang , envdict = dict ( os . environ ) )
chroot_path , cmd , lang = lang , envdict = dict ( os . environ ) )
# замена выполения команд: вместо оычного запуска - запуск через
# замена выполения команд: вместо оычного запуска - запуск через
# /usr/bin/chroot
# /usr/bin/chroot
@ -264,7 +249,7 @@ class Builder(Update):
def _regenCache_process ( self , progname , repname , cpu_num ) :
def _regenCache_process ( self , progname , repname , cpu_num ) :
chroot_path = self . clVars . Get ( ' builder.cl_builder_path ' )
chroot_path = self . clVars . Get ( ' builder.cl_builder_path ' )
return process( " /usr/bin/chroot " , chroot_path ,
return self . chroot_ process( chroot_path ,
progname , " --repo= %s " % repname , " --update " ,
progname , " --repo= %s " % repname , " --update " ,
" --jobs= %s " % cpu_num , stderr = STDOUT )
" --jobs= %s " % cpu_num , stderr = STDOUT )
@ -283,6 +268,20 @@ class Builder(Update):
mainlog = self . clVars . Get ( ' core.cl_log_path ' )
mainlog = self . clVars . Get ( ' core.cl_log_path ' )
return path . join ( mainlog , logname )
return path . join ( mainlog , logname )
def chrootize ( self , chroot_path , cmd ) :
"""
: param chroot_path :
: param cmd :
: return :
"""
arch = self . clVars . Get ( ' os_builder_arch_machine ' )
local_arch = self . clVars . Get ( ' os_arch_machine ' )
# упрощенная проверка так как только 64 может собирать 32
if arch != local_arch :
return Linux32 ( Chroot ( chroot_path , cmd ) )
else :
return Chroot ( chroot_path , cmd )
def emerge_ask ( self , pretend , * params ) :
def emerge_ask ( self , pretend , * params ) :
"""
"""
Вывести информацию о б обновлении
Вывести информацию о б обновлении
@ -292,11 +291,11 @@ class Builder(Update):
packages = [ x for x in params if not x . startswith ( " - " ) ]
packages = [ x for x in params if not x . startswith ( " - " ) ]
chroot_path = self . clVars . Get ( ' cl_builder_path ' )
chroot_path = self . clVars . Get ( ' cl_builder_path ' )
logfile = self . _get_log_file ( )
logfile = self . _get_log_file ( )
with EmergeParser ( ChrootEmergeCommand ( chroot_path ,
with EmergeParser ( self . chrootize ( chroot_path , EmergeCommand (
list ( packages ) ,
list ( packages ) ,
emerge_default_opts = deo ,
emerge_default_opts = deo ,
extra_params = param ,
extra_params = param ,
logfile = logfile ) ) as emerge :
logfile = logfile ) ) ) as emerge :
try :
try :
emerge . question . action = lambda x : False
emerge . question . action = lambda x : False
emerge . run ( )
emerge . run ( )
@ -330,10 +329,10 @@ class Builder(Update):
deo = self . clVars . Get ( ' cl_emerge_default_opts ' )
deo = self . clVars . Get ( ' cl_emerge_default_opts ' )
chroot_path = self . clVars . Get ( ' cl_builder_path ' )
chroot_path = self . clVars . Get ( ' cl_builder_path ' )
logfile = self . _get_log_file ( )
logfile = self . _get_log_file ( )
with EmergeParser (
with EmergeParser ( self . chrootize ( chroot_path , EmergeCommand (
ChrootEmergeCommand ( chroot_path , [ " --depclean " , " --ask=y " ] ,
[ " --depclean " , " --ask=y " ] ,
logfile = logfile ,
logfile = logfile ,
emerge_default_opts = deo ) ) as emerge :
emerge_default_opts = deo ) ) ) as emerge :
try :
try :
emerge . question . action = lambda x : False
emerge . question . action = lambda x : False
emerge . run ( )
emerge . run ( )
@ -363,10 +362,7 @@ class Builder(Update):
if not cmdpath :
if not cmdpath :
return " skip "
return " skip "
#raise BuilderError(_("The %s tool is not found")%command)
p = self . chroot_process ( builder_path , cmdpath , * params , stderr = STDOUT )
p = process ( " /usr/bin/chroot " ,
* ( [ builder_path , cmdpath ] + list ( params ) ) ,
stderr = STDOUT )
with writeFile ( self . _get_log_file ( ) ) as f :
with writeFile ( self . _get_log_file ( ) ) as f :
f . write ( p . read ( ) )
f . write ( p . read ( ) )
if p . failed ( ) :
if p . failed ( ) :
@ -459,9 +455,9 @@ class Builder(Update):
: return : список пакетов
: return : список пакетов
"""
"""
if atom not in self . pretend_package_list :
if atom not in self . pretend_package_list :
emerge = EmergeParser ( Chroot EmergeCommand(
with EmergeParser ( self . chrootize ( chroot_path , EmergeCommand(
chroot_path , [ atom ] , extra_params = [ ' -pv ' , ' --ask=n ' ] ,
[ atom ] , extra_params = [ ' -pv ' , ' --ask=n ' ] ,
logfile = logfile ) )
logfile = logfile ) ) ) as emerge :
emerge . question . default_answer = " n "
emerge . question . default_answer = " n "
emerge . run ( )
emerge . run ( )
self . pretend_package_list [ atom ] = list ( emerge . install_packages . list )
self . pretend_package_list [ atom ] = list ( emerge . install_packages . list )
@ -539,9 +535,9 @@ class Builder(Update):
self . startTask ( _ ( " Fetching binary packages and sources tarballs " ) %
self . startTask ( _ ( " Fetching binary packages and sources tarballs " ) %
[ x for x in package_list if x [ ' PN ' ] == drv_name ] [ 0 ] )
[ x for x in package_list if x [ ' PN ' ] == drv_name ] [ 0 ] )
ef = EmergeFetcher ( Chroot EmergeCommand(
ef = EmergeFetcher ( self . chrootize ( builder_path , EmergeCommand(
builder_path , [ " = %s " % x for x in package_list ] ,
[ " = %s " % x for x in package_list ] ,
extra_params = [ " -Of " , " --ask=n " ] , logfile = " %s .2 " % logfile ) )
extra_params = [ " -Of " , " --ask=n " ] , logfile = " %s .2 " % logfile ) ) )
try :
try :
for package in ef :
for package in ef :
pkg_name = str ( package )
pkg_name = str ( package )
@ -656,10 +652,8 @@ class Builder(Update):
: return :
: return :
"""
"""
eselect_command = " /usr/bin/eselect "
eselect_command = " /usr/bin/eselect "
p = process ( " /usr/bin/chroot " ,
p = self . chroot_process ( builder_path , eselect_command , " --colour=yes " ,
* [ builder_path , eselect_command , " --colour=yes " ,
" news " , " read " , " new " , stderr = STDOUT )
" news " , " read " , " new " ] ,
stderr = STDOUT )
self . printPre ( convert_console_to_xml (
self . printPre ( convert_console_to_xml (
p . read ( ) ) . replace ( " " , " " ) )
p . read ( ) ) . replace ( " " , " " ) )
return True
return True
@ -689,10 +683,9 @@ class Builder(Update):
logfile = self . _get_log_file ( )
logfile = self . _get_log_file ( )
if not cmd_path :
if not cmd_path :
raise BuilderError ( _ ( " Failed to find the %s command " ) % cmd )
raise BuilderError ( _ ( " Failed to find the %s command " ) % cmd )
with EmergeParser (
with EmergeParser ( self . chrootize ( builder_path , CommandExecutor (
ChrootCommandExecutor ( builder_path ,
cmd_path , params ,
cmd_path , params ,
logfile = logfile ) ) as emerge :
logfile = logfile ) ) ) as emerge :
self . _startEmerging ( emerge )
self . _startEmerging ( emerge )
return True
return True
@ -704,8 +697,8 @@ class Builder(Update):
logfile = self . _get_log_file ( )
logfile = self . _get_log_file ( )
if not cmd_path :
if not cmd_path :
raise BuilderError ( _ ( " Failed to find the %s command " ) % cmd )
raise BuilderError ( _ ( " Failed to find the %s command " ) % cmd )
with EmergeParser ( Chroot CommandExecutor(
with EmergeParser ( self . chrootize ( builder_path , CommandExecutor(
builder_path , cmd_path , params , logfile = logfile ) ) as emerge :
builder_path , cmd_path , params , logfile = logfile ) ) ) as emerge :
revdep = RevdepPercentBlock ( emerge )
revdep = RevdepPercentBlock ( emerge )
self . addProgress ( )
self . addProgress ( )
revdep . add_observer ( self . setProgress )
revdep . add_observer ( self . setProgress )
@ -738,9 +731,8 @@ class Builder(Update):
for perc in p . progress ( ) :
for perc in p . progress ( ) :
self . setProgress ( perc )
self . setProgress ( perc )
else :
else :
p = process (
p = self . chroot_process (
" /usr/bin/chroot " , chroot_path ,
chroot_path , layman , " -s " , repname , stderr = STDOUT )
layman , " -s " , repname , stderr = STDOUT )
if p . failed ( ) :
if p . failed ( ) :
raise BuilderError (
raise BuilderError (
_ ( " Failed to update the {rname} repository " ) . format (
_ ( " Failed to update the {rname} repository " ) . format (
@ -796,3 +788,120 @@ class Builder(Update):
self . clVars . Set ( ' update.cl_update_branch_name ' ,
self . clVars . Set ( ' update.cl_update_branch_name ' ,
self . clVars . Get ( ' builder.cl_builder_branch_name ' ) )
self . clVars . Get ( ' builder.cl_builder_branch_name ' ) )
return True
return True
def isohybrid ( self , image_file ) :
"""
Преобразовать ISO образ в гибридный
: param image_file : iso образ
: return :
"""
isohybrid = getProgPath ( " /usr/bin/isohybrid " )
if not isohybrid :
raise BuilderError ( _ ( " Isohybrid utility not found " ) )
if not path . exists ( image_file ) :
raise BuilderError ( _ ( " Image not found " ) )
if self . clVars . Get ( ' os_builder_arch_machine ' ) == ' x86_64 ' :
cmd = [ isohybrid , " --uefi " , image_file ]
else :
cmd = [ isohybrid , image_file ]
isohybrid_process = process ( * cmd )
return isohybrid_process . success ( )
def _list_file ( self , iso_file ) :
"""
. list файл по iso файлу
: param iso_file :
: return :
"""
if iso_file . endswith ( " .iso " ) :
return " %s .list " % iso_file [ : - 4 ]
else :
return " %s .list " % iso_file
def _digest_file ( self , iso_file ) :
return " %s .DIGEST " % iso_file
def create_package_list ( self , chroot , iso_file ) :
"""
Создает список установленных пакетов в chroot директории и сохраняет в
iso_file
: return :
"""
pkgdir = path . join ( chroot , ' var/db/pkg ' )
list_file = self . _list_file ( iso_file )
with writeFile ( list_file ) as f :
f . write ( " \n " . join ( sorted (
x . atom for x in InstalledPackageInfo . get_install_packages (
pkg_dir = pkgdir ) ) ) )
return True
def create_digest ( self , isofile ) :
"""
Создать контрольную сумму для файла
"""
template = """ # %(alg)s HASH \n %(digest)s %(filename)s \n """
digestfile = self . _digest_file ( isofile )
try :
with writeFile ( digestfile ) as f :
f . writelines ( map ( lambda x : template % {
' alg ' : x . upper ( ) ,
' digest ' :
process ( " %s sum " % x , isofile ) . read ( ) . partition ( ' ' ) [ 0 ] ,
' filename ' : path . basename ( isofile ) } , [ " md5 " , " sha1 " ] ) )
except ( IOError , OSError ) :
return False
return True
@property
def chroot_process ( self ) :
chroot_cmd = getProgPath ( " /usr/bin/chroot " )
if not chroot_cmd :
raise BuilderError ( _ ( " Chroot command not found " ) )
arch = self . clVars . Get ( ' builder.os_builder_arch_machine ' )
local_arch = self . clVars . Get ( ' os_arch_machine ' )
process_func = process
if arch != local_arch :
process_func = partial ( process_func , " /usr/bin/linux32 " )
return partial ( process_func , chroot_cmd )
def emerge ( self , builder_path , use , * params ) :
"""
Выполнить сборку пакета
"""
deo = self . clVars . Get ( ' cl_emerge_default_opts ' )
extra_params = [ x for x in params if x . startswith ( " - " ) ]
packages = [ x for x in params if not x . startswith ( " - " ) ]
with EmergeParser ( self . chrootize ( builder_path , EmergeCommand (
packages , emerge_default_opts = deo ,
extra_params = extra_params , use = use ) ) ) as emerge :
try :
emerge . question . action = lambda x : False
emerge . run ( )
if not emerge . install_packages . list :
return True
except EmergeError :
self . emerge_cache . drop_cache ( " Emerge error " )
self . _display_error ( emerge . prepare_error )
raise
self . _startEmerging ( emerge )
return True
def remove_list_digest ( self , isofile ) :
for fn in ( self . _digest_file ( isofile ) , self . _list_file ( isofile ) ) :
if path . exists ( fn ) :
os . unlink ( fn )
return True
def recount_files ( self , builder_path , exclude_files ) :
"""
Посчитать файлы в собираемой системе и исключить exclude файлы
: param builder_path :
: param exclude_files :
: return :
"""
all_count = countFiles ( builder_path )
exclude_sum = sum ( countFiles ( path . join ( builder_path , x ) )
for x in exclude_files )
self . clVars . Set ( ' os_builder_linux_files ' , str ( all_count - exclude_sum ) ,
force = True )
return True