diff --git a/pym/builder/builder.py b/pym/builder/builder.py index b9ff10a..8964ad3 100644 --- a/pym/builder/builder.py +++ b/pym/builder/builder.py @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from collections import OrderedDict from itertools import chain @@ -20,13 +21,11 @@ import re import sys import time import stat -import glob from calculate.core.server.gen_pid import search_worked_process2 from calculate.core.setup_package import ChainProgressTemplate from calculate.lib.cl_template import templateFunction, SystemIni from calculate.lib.datavars import DataVars, Variable -from calculate.lib.utils.colortext import (TextState, get_color_print, - convert_console_to_xml) +from calculate.lib.utils.colortext import (TextState, get_color_print) from calculate.lib.utils.common import CmdlineParams from calculate.lib.utils.files import ( pathJoin, PercentProgress, getProgPath, process, STDOUT, removeDir, @@ -40,9 +39,9 @@ from calculate.lib.utils.portage import (Layman, EmergeLog, EmergeLogNamedTask, get_packages_files_directory, get_manifest_files_directory, PackageList, VDB_PATH, - hide_package, unhide_package, - LibraryProviders, EmergePackage, - getInstalledAtom) + hide_packages, unhide_packages, + LibraryProviders, + PackageError, getInstalledAtom) from calculate.update.emerge_parser import (EmergeParser, EmergeError, EmergeCommand, Chroot, Linux32, CommandExecutor, @@ -88,6 +87,8 @@ class Builder(Update): self.update_map = {} self.color_print = get_color_print() self.emerge_cache = EmergeCache() + self.pkgnum = None + self.pkgnummax = None def mount_target(self, target): dir_distro = target.convertToDirectory() @@ -1365,7 +1366,8 @@ class Builder(Update): cache_list = ("/var/calculate/tmp/%s.checkdep" % self.clVars.Get("cl_builder_id_path")) task_list = list(chain(*[list(getInstalledAtom(x, prefix=builder_path)) - for x in readLinesFile(cache_list)])) + for x in readLinesFile(cache_list, + grab=True)])) task_list = list(task.list) + task_list vdb_path = "var/db/.pkg" @@ -1376,7 +1378,7 @@ class Builder(Update): except (IOError, OSError): raise BuilderError(_("Failed to hide package database")) - automagic = {} + automagic = OrderedDict() automagic_waste = {} automagic_skip = {} try: @@ -1393,13 +1395,20 @@ class Builder(Update): yield pkg, list(set(get_all_reqs(pkg))) - check_data = {x: y for x, y in get_check_data() if y} + check_data = {x: y for x, y in get_check_data() + if x["CATEGORY"] != "virtual"} - for i, data in enumerate(check_data.items()): + for i, data in enumerate(sorted(check_data.items(), + key=lambda x:x[0])): package, required_pkgs = data self.startTask("Check (%d of %d) %s" % ( i + 1, len(check_data), package)) + + if not required_pkgs: + self.endTask(True) + continue + pretend = list(self.pretend_emerge_list(builder_path, package)) if not pretend: self.endTask(False) @@ -1432,6 +1441,18 @@ class Builder(Update): shutil.move(hide_vdb_path, real_vdb_path) except (IOError, OSError): raise BuilderError(_("Failed to unhide package database")) + self._report_automagic(automagic, automagic_skip, automagic_waste) + self._rebuild_automagic(automagic, automagic_skip, builder_path) + return True + + def _report_automagic(self, automagic, automagic_skip, automagic_waste): + """ + Сообщить информацию о вычисленных автоматических зависимостей + :param automagic: + :param automagic_skip: + :param automagic_waste: + :return: + """ if automagic: for pkg, reqs in automagic.items(): skip_req_pkgs = automagic_skip.get(pkg, []) @@ -1449,29 +1470,68 @@ class Builder(Update): "Auto depends for %s package by %s obsoleted" % ( pkg, ",".join(waste_pkgs))) - for pkg, reqs in automagic.items(): - hidden_pkgs = [] - skip_req_pkgs = automagic_skip.get(pkg, []) - try: + def _rebuild_automagic(self, automagic, automagic_skip, builder_path): + """ + Пересобрать пакеты с автоматическими зависимостями + :param automagic: + :param automagic_skip: + :param builder_path: + :return: + """ + automagic_log_dn = "/var/log/calculate/automagic" + makeDirectory(automagic_log_dn) + self.pkgnum = 0 + self.pkgnummax = len(automagic) + try: + for pkg, reqs in automagic.items(): + skip_req_pkgs = automagic_skip.get(pkg, []) + self.pkgnum += 1 if not reqs: if skip_req_pkgs: - self.printWARNING(_("Skip remerge %s package") % pkg) + self.printWARNING( + _("Skip ({num} of {nummax}) remerge " + "{package} package").format(num=self.pkgnum, + nummax=self.pkgnummax, + package=str(pkg))) continue - for req in reqs: - hide_package(req, prefix=builder_path) - hidden_pkgs.append(req) - - self.emerge(builder_path, "-O", "=%s" % pkg) - - rebuild_log_file = ("/var/log/calculate/automagic-%s.log" % - self.clVars.Get("cl_builder_id_path")) - with open(rebuild_log_file, 'a') as f: - f.write("{package}, hidden: {hidden_pkgs}\n".format( - package=pkg, - hidden_pkgs=",".join(str(x) for x in hidden_pkgs) - )) - finally: - for unhidepkg in reversed(hidden_pkgs): - unhide_package(unhidepkg, prefix=builder_path) + try: + hide_packages(*reqs, prefix=builder_path) + + self.emerge(builder_path, "-O", "=%s" % pkg) + + rebuild_log_file = ("/var/log/calculate/automagic-%s.log" % + self.clVars.Get("cl_builder_id_path")) + with open(rebuild_log_file, 'a') as f: + f.write("{package}, hidden: {hidden_pkgs}\n".format( + package=pkg, + hidden_pkgs=",".join(str(x) for x in reqs))) + except EmergeError: + old_logfile = self._get_log_file() + pkg_path = str(pkg).replace("/", "_") + new_logfile = '%s/%s-%s.log' % ( + automagic_log_dn, self.clVars.Get('cl_builder_id_path'), + pkg_path) + try: + os.rename(old_logfile, new_logfile) + except (OSError, IOError) as e: + self.printERROR("{message}: {error}".format( + message=_("Failed to save build log"), + error=str(e))) + self.printERROR( + _("Failed to merge {package} without " + "{hidden_pkgs}").format( + package=str(pkg), + hidden_pkgs=",".join(str(x) for x in reqs))) + raise + except PackageError as e: + raise BuilderError(str(e)) + finally: + try: + unhide_packages(prefix=builder_path, force=True) + except PackageError as e: + raise BuilderError(str(e)) + finally: + self.pkgnummax = None + self.pkgnum = None return True diff --git a/pym/builder/utils/cl_builder_update.py b/pym/builder/utils/cl_builder_update.py index 2f93fa9..ddf4d8f 100644 --- a/pym/builder/utils/cl_builder_update.py +++ b/pym/builder/utils/cl_builder_update.py @@ -18,10 +18,11 @@ import sys from calculate.core.server.func import Action, Tasks from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate from calculate.lib.cl_template import TemplatesError -from calculate.lib.utils.files import FilesError +from calculate.lib.utils.files import FilesError, isEmptyFile, readFile from calculate.update.emerge_parser import EmergeError from calculate.update.update import UpdateError from ..datavars import BuilderError +from os import path from calculate.lib.utils.git import GitError from calculate.lib.utils.portage import (EmergeLogNamedTask, PackageList, EmergeLog) @@ -52,6 +53,21 @@ class BuilderConditions(object): return func + @staticmethod + def force_automagic(Get): + cache_list = ("/var/calculate/tmp/%s.checkdep" % + Get("cl_builder_id_path")) + return not isEmptyFile(cache_list, grab=True) + + @staticmethod + def force_preserved(Get): + pfile = path.join(Get("cl_builder_path"), + "var/lib/portage/preserved_libs_registry") + content = readFile(pfile).strip() + if not content or content == '{}': + return False + else: + return True class ClBuilderUpdateAction(Action): """ @@ -289,8 +305,10 @@ class ClBuilderUpdateAction(Action): 'message': __('Updating preserved libraries'), 'method': 'Builder.emerge(cl_builder_path,"",' '"@preserved-rebuild")', - 'condition': BuilderConditions.was_installed( - '.*', EmergeMark.PreservedLibs), + 'condition': lambda Get: ( + BuilderConditions.was_installed( + '.*', EmergeMark.PreservedLibs) or + BuilderConditions.force_preserved(Get)), 'decoration': 'Builder.update_task("%s")' % EmergeMark.PreservedLibs }, @@ -305,34 +323,41 @@ class ClBuilderUpdateAction(Action): 'decoration': 'Builder.update_task("%s")' % EmergeMark.RevdepRebuild }, - {'name': 'update_world:dispatch_conf_end', - 'message': __("Updating configuration files"), - 'method': 'Builder.dispatchConf(None,cl_builder_path)', - 'condition': lambda Get: Get('cl_dispatch_conf') != 'skip' - }, - {'name': 'update_world:binary_cleaning', - 'message': __("Cleaning the binary repository"), - 'method': 'Builder.binaryCleaning()', - 'condition': lambda Get: Get( - 'cl_builder_binary_set') == "off" - }, ], 'depend': Tasks.has("update_other") }, - {'name': 'update_world:check_automagic_group', + {'name': 'update_other:check_automagic_group', 'group': __("Check for auto depends"), 'tasks': [ {'name': 'check_automagic', 'method': 'Builder.check_automagic(cl_builder_path)', 'condition': lambda Get: ( Get('builder.cl_builder_check_automagic_set') == 'on' - and BuilderConditions.was_installed( - '.*', EmergeMark.Automagic)(Get)), + and (BuilderConditions.was_installed( + '.*', EmergeMark.Automagic)(Get) or + BuilderConditions.force_automagic(Get))), 'decoration': 'Builder.update_task("%s")' % EmergeMark.Automagic }, ] }, + {'name': 'update_world:update_configures', + 'group': __("Completion of the system update"), + 'tasks': [ + {'name': 'update_world:dispatch_conf_end', + 'message': __("Updating configuration files"), + 'method': 'Builder.dispatchConf(None,cl_builder_path)', + 'condition': lambda Get: Get('cl_dispatch_conf') != 'skip' + }, + {'name': 'update_world:binary_cleaning', + 'message': __("Cleaning the binary repository"), + 'method': 'Builder.binaryCleaning()', + 'condition': lambda Get: Get( + 'cl_builder_binary_set') == "off" + }, + ], + 'depend': Tasks.has("update_other") + }, {'name': 'update_other:reading_news_group', 'group': __("Last news:"), 'tasks': [