calculate
/
calculate-utils-3-update
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Доработана пересборка пакетов с автоматическими зависимостями

Browse Source 
Добавлены дополнительные условия выполнения preserved-rebuild
Хирецкий Михаил 7 years ago
parent 9d3dc8abcc
commit 6628d1f927
2 changed files with 133 additions and 48 deletions
Show Stats Download Patch File Download Diff File

122
pym/builder/builder.py
Unescape View File

@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import OrderedDict
from itertools import chain
@ -20,13 +21,11 @@ import re
import sys
import time
import stat
import glob
from calculate.core.server.gen_pid import search_worked_process2
from calculate.core.setup_package import ChainProgressTemplate
from calculate.lib.cl_template import templateFunction, SystemIni
from calculate.lib.datavars import DataVars, Variable
from calculate.lib.utils.colortext import (TextState, get_color_print,
convert_console_to_xml)
from calculate.lib.utils.colortext import (TextState, get_color_print)
from calculate.lib.utils.common import CmdlineParams
from calculate.lib.utils.files import (
pathJoin, PercentProgress, getProgPath, process, STDOUT, removeDir,
@ -40,9 +39,9 @@ from calculate.lib.utils.portage import (Layman, EmergeLog, EmergeLogNamedTask,
get_packages_files_directory,
get_manifest_files_directory,
PackageList, VDB_PATH,
hide_package, unhide_package,
LibraryProviders, EmergePackage,
getInstalledAtom)
hide_packages, unhide_packages,
LibraryProviders,
PackageError, getInstalledAtom)
from calculate.update.emerge_parser import (EmergeParser,
EmergeError, EmergeCommand, Chroot,
Linux32, CommandExecutor,
@ -88,6 +87,8 @@ class Builder(Update):
self.update_map = {}
self.color_print = get_color_print()
self.emerge_cache = EmergeCache()
self.pkgnum = None
self.pkgnummax = None
def mount_target(self, target):
dir_distro = target.convertToDirectory()
@ -1365,7 +1366,8 @@ class Builder(Update):
cache_list = ("/var/calculate/tmp/%s.checkdep" %
self.clVars.Get("cl_builder_id_path"))
task_list = list(chain(*[list(getInstalledAtom(x, prefix=builder_path))
for x in readLinesFile(cache_list)]))
for x in readLinesFile(cache_list,
grab=True)]))
task_list = list(task.list) + task_list
vdb_path = "var/db/.pkg"
@ -1376,7 +1378,7 @@ class Builder(Update):
except (IOError, OSError):
raise BuilderError(_("Failed to hide package database"))
automagic = {}
automagic = OrderedDict()
automagic_waste = {}
automagic_skip = {}
try:
@ -1393,13 +1395,20 @@ class Builder(Update):
yield pkg, list(set(get_all_reqs(pkg)))
check_data = {x: y for x, y in get_check_data() if y}
check_data = {x: y for x, y in get_check_data()
if x["CATEGORY"] != "virtual"}
for i, data in enumerate(check_data.items()):
for i, data in enumerate(sorted(check_data.items(),
key=lambda x:x[0])):
package, required_pkgs = data
self.startTask("Check (%d of %d) %s" % (
i + 1, len(check_data), package))
if not required_pkgs:
self.endTask(True)
continue
pretend = list(self.pretend_emerge_list(builder_path, package))
if not pretend:
self.endTask(False)
@ -1432,6 +1441,18 @@ class Builder(Update):
shutil.move(hide_vdb_path, real_vdb_path)
except (IOError, OSError):
raise BuilderError(_("Failed to unhide package database"))
self._report_automagic(automagic, automagic_skip, automagic_waste)
self._rebuild_automagic(automagic, automagic_skip, builder_path)
return True
def _report_automagic(self, automagic, automagic_skip, automagic_waste):
"""
Сообщить информацию о вычисленных автоматических зависимостей
:param automagic:
:param automagic_skip:
:param automagic_waste:
:return:
"""
if automagic:
for pkg, reqs in automagic.items():
skip_req_pkgs = automagic_skip.get(pkg, [])
@ -1449,29 +1470,68 @@ class Builder(Update):
"Auto depends for %s package by %s obsoleted" % (
pkg, ",".join(waste_pkgs)))
for pkg, reqs in automagic.items():
hidden_pkgs = []
skip_req_pkgs = automagic_skip.get(pkg, [])
try:
def _rebuild_automagic(self, automagic, automagic_skip, builder_path):
"""
Пересобрать пакеты с автоматическими зависимостями
:param automagic:
:param automagic_skip:
:param builder_path:
:return:
"""
automagic_log_dn = "/var/log/calculate/automagic"
makeDirectory(automagic_log_dn)
self.pkgnum = 0
self.pkgnummax = len(automagic)
try:
for pkg, reqs in automagic.items():
skip_req_pkgs = automagic_skip.get(pkg, [])
self.pkgnum += 1
if not reqs:
if skip_req_pkgs:
self.printWARNING(_("Skip remerge %s package") % pkg)
self.printWARNING(
_("Skip ({num} of {nummax}) remerge "
"{package} package").format(num=self.pkgnum,
nummax=self.pkgnummax,
package=str(pkg)))
continue
for req in reqs:
hide_package(req, prefix=builder_path)
hidden_pkgs.append(req)
self.emerge(builder_path, "-O", "=%s" % pkg)
rebuild_log_file = ("/var/log/calculate/automagic-%s.log" %
self.clVars.Get("cl_builder_id_path"))
with open(rebuild_log_file, 'a') as f:
f.write("{package}, hidden: {hidden_pkgs}\n".format(
package=pkg,
hidden_pkgs=",".join(str(x) for x in hidden_pkgs)
))
finally:
for unhidepkg in reversed(hidden_pkgs):
unhide_package(unhidepkg, prefix=builder_path)
try:
hide_packages(*reqs, prefix=builder_path)
self.emerge(builder_path, "-O", "=%s" % pkg)
rebuild_log_file = ("/var/log/calculate/automagic-%s.log" %
self.clVars.Get("cl_builder_id_path"))
with open(rebuild_log_file, 'a') as f:
f.write("{package}, hidden: {hidden_pkgs}\n".format(
package=pkg,
hidden_pkgs=",".join(str(x) for x in reqs)))
except EmergeError:
old_logfile = self._get_log_file()
pkg_path = str(pkg).replace("/", "_")
new_logfile = '%s/%s-%s.log' % (
automagic_log_dn, self.clVars.Get('cl_builder_id_path'),
pkg_path)
try:
os.rename(old_logfile, new_logfile)
except (OSError, IOError) as e:
self.printERROR("{message}: {error}".format(
message=_("Failed to save build log"),
error=str(e)))
self.printERROR(
_("Failed to merge {package} without "
"{hidden_pkgs}").format(
package=str(pkg),
hidden_pkgs=",".join(str(x) for x in reqs)))
raise
except PackageError as e:
raise BuilderError(str(e))
finally:
try:
unhide_packages(prefix=builder_path, force=True)
except PackageError as e:
raise BuilderError(str(e))
finally:
self.pkgnummax = None
self.pkgnum = None
return True

59
pym/builder/utils/cl_builder_update.py
Unescape View File

@ -18,10 +18,11 @@ import sys
from calculate.core.server.func import Action, Tasks
from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
from calculate.lib.cl_template import TemplatesError
from calculate.lib.utils.files import FilesError
from calculate.lib.utils.files import FilesError, isEmptyFile, readFile
from calculate.update.emerge_parser import EmergeError
from calculate.update.update import UpdateError
from ..datavars import BuilderError
from os import path
from calculate.lib.utils.git import GitError
from calculate.lib.utils.portage import (EmergeLogNamedTask,
PackageList, EmergeLog)
@ -52,6 +53,21 @@ class BuilderConditions(object):
return func
@staticmethod
def force_automagic(Get):
cache_list = ("/var/calculate/tmp/%s.checkdep" %
Get("cl_builder_id_path"))
return not isEmptyFile(cache_list, grab=True)
@staticmethod
def force_preserved(Get):
pfile = path.join(Get("cl_builder_path"),
"var/lib/portage/preserved_libs_registry")
content = readFile(pfile).strip()
if not content or content == '{}':
return False
else:
return True
class ClBuilderUpdateAction(Action):
"""
@ -289,8 +305,10 @@ class ClBuilderUpdateAction(Action):
'message': __('Updating preserved libraries'),
'method': 'Builder.emerge(cl_builder_path,"",'
'"@preserved-rebuild")',
'condition': BuilderConditions.was_installed(
'.*', EmergeMark.PreservedLibs),
'condition': lambda Get: (
BuilderConditions.was_installed(
'.*', EmergeMark.PreservedLibs) or
BuilderConditions.force_preserved(Get)),
'decoration': 'Builder.update_task("%s")' %
EmergeMark.PreservedLibs
},
@ -305,34 +323,41 @@ class ClBuilderUpdateAction(Action):
'decoration': 'Builder.update_task("%s")' %
EmergeMark.RevdepRebuild
},
{'name': 'update_world:dispatch_conf_end',
'message': __("Updating configuration files"),
'method': 'Builder.dispatchConf(None,cl_builder_path)',
'condition': lambda Get: Get('cl_dispatch_conf') != 'skip'
},
{'name': 'update_world:binary_cleaning',
'message': __("Cleaning the binary repository"),
'method': 'Builder.binaryCleaning()',
'condition': lambda Get: Get(
'cl_builder_binary_set') == "off"
},
],
'depend': Tasks.has("update_other")
},
{'name': 'update_world:check_automagic_group',
{'name': 'update_other:check_automagic_group',
'group': __("Check for auto depends"),
'tasks': [
{'name': 'check_automagic',
'method': 'Builder.check_automagic(cl_builder_path)',
'condition': lambda Get: (
Get('builder.cl_builder_check_automagic_set') == 'on'
and BuilderConditions.was_installed(
'.*', EmergeMark.Automagic)(Get)),
and (BuilderConditions.was_installed(
'.*', EmergeMark.Automagic)(Get) or
BuilderConditions.force_automagic(Get))),
'decoration': 'Builder.update_task("%s")' %
EmergeMark.Automagic
},
]
},
{'name': 'update_world:update_configures',
'group': __("Completion of the system update"),
'tasks': [
{'name': 'update_world:dispatch_conf_end',
'message': __("Updating configuration files"),
'method': 'Builder.dispatchConf(None,cl_builder_path)',
'condition': lambda Get: Get('cl_dispatch_conf') != 'skip'
},
{'name': 'update_world:binary_cleaning',
'message': __("Cleaning the binary repository"),
'method': 'Builder.binaryCleaning()',
'condition': lambda Get: Get(
'cl_builder_binary_set') == "off"
},
],
'depend': Tasks.has("update_other")
},
{'name': 'update_other:reading_news_group',
'group': __("Last news:"),
'tasks': [

Write Preview
Loading…
Cancel
Save