TG-606

проверка для update

.gitignore

@@ -0,0 +1,6 @@
+revert_changes_to_vmachine
+push_to_vmachine*
+.vscode
+*.pyc
+*.pyo
+*.bak

pym/update/emerge_parser.py

@@ -55,7 +55,7 @@ class EmergeNeedRootError(EmergeError):
     pass
 
 
-class CommandExecutor(object):
+class CommandExecutor():
     """
     Запуск программы для объекта Emerge
     """
@@ -75,17 +75,23 @@ class CommandExecutor(object):
     def get_command(self):
         return [self.cmd] + list(self.params)
 
+    #TODO make sure pexpect encoding doesn't mess up anything
+    #if it does, instead put log file as binary maybe?
     def execute(self):
         if self.child is None:
             command_data = self.get_command()
             self.child = pexpect.spawn(command_data[0], command_data[1:],
                                        logfile=open(self.logfile, 'w'),
-                                       env=self.env, cwd=self.cwd, timeout=None)
+                                       env=self.env, cwd=self.cwd, timeout=None,
+                                       encoding="UTF-8", codec_errors='ignore')
         return self.child
 
     def close(self):
         if self.child is not None:
-            self.child.close()
+            try:
+                self.child.close()
+            except pexpect.ExceptionPexpect:
+                self.child.close(force=True)
             self.child = None
 
     def success(self):
@@ -133,7 +139,7 @@ class EmergeCommand(CommandExecutor):
             env.update(env_update)
 
         params = self.default_params + extra_params + packages
-        super(EmergeCommand, self).__init__(self.emerge_cmd, params=params,
+        super().__init__(self.emerge_cmd, params=params,
                                             env=env, cwd=cwd, logfile=logfile)
 
 
@@ -172,7 +178,7 @@ def Linux32(obj):
     return obj
 
 
-class InfoBlockInterface(object):
+class InfoBlockInterface():
     """
     Интерфейс для информационного блока
     """
@@ -281,7 +287,7 @@ class InstallPackagesBlock(EmergeInformationBlock):
     re_blocks = re.compile(r"\[{c}blocks{c} {c}b".format(c=_color_block))
 
     def get_data(self, match):
-        super(InstallPackagesBlock, self).get_data(match)
+        super().get_data(match)
         list_block = XmlConverter().transform(self.result).split('\n')
         self.list = PackageList(map(EmergeUpdateInfo, list_block))
         self.remove_list = PackageList(map(EmergeRemoveInfo, list_block))
@@ -511,7 +517,7 @@ class NotifierInformationBlock(EmergeInformationBlock):
     Информационный блок поддерживающий observing
     """
     def __init__(self, parent):
-        super(NotifierInformationBlock, self).__init__(parent)
+        super().__init__(parent)
         self.observers = []
 
     def get_data(self, match):
@@ -716,7 +722,7 @@ class EmergeParser(InfoBlockInterface):
 
         while True:
             index = child.expect_exact(self.elements.keys())
-            element = self.elements.values()[index]
+            element = list(self.elements.values())[index]
             element.get_block(child)
             if element.action:
                 if element.action(child) is False:
@@ -732,7 +738,7 @@ class EmergeParser(InfoBlockInterface):
         self.close()
 
 
-class MtimeCheckvalue(object):
+class MtimeCheckvalue():
     def __init__(self, *fname):
         self.fname = fname
 
@@ -754,10 +760,10 @@ class MtimeCheckvalue(object):
 
 class Md5Checkvalue(MtimeCheckvalue):
     def value_func(self, fn):
-        return hashlib.md5(readFile(fn)).hexdigest()
+        return hashlib.md5(readFile(fn, binary=True)).hexdigest()
 
 
-class GitCheckvalue(object):
+class GitCheckvalue():
     def __init__(self, git, rpath):
         self.rpath = rpath
         self.git = git
@@ -768,7 +774,7 @@ class GitCheckvalue(object):
                 yield self.rpath, self.git.getCurrentCommit(self.rpath)
 
 
-class EmergeCache(object):
+class EmergeCache():
     """
     Кэш пакетов
     """

pym/update/profile.py

@@ -21,14 +21,14 @@ import shutil
 from calculate.lib.utils.files import (listDirectory, readFile, readLinesFile,
                                        makeDirectory, removeDir)
 from calculate.lib.utils.git import Git
-from update import UpdateError
+from .update import UpdateError
 
 from calculate.lib.cl_lang import setLocalTranslate, _
 setLocalTranslate('cl_update3', sys.modules[__name__])
 
 DEFAULT_BRANCH = Git.Reference.Master
 
-class RepositoryStorageInterface(object):
+class RepositoryStorageInterface():
     def __iter__(self):
         raise StopIteration
 
@@ -141,7 +141,7 @@ class RepositoryStorageSet(RepositoryStorageInterface):
     def __repr__(self):
         return "Repository set"
 
-class Profile(object):
+class Profile():
     """
     Профиль репозитория
     """
@@ -151,14 +151,22 @@ class Profile(object):
         self.repository = repository
         self.profile = profile
         self.arch = arch
+        self._path = None
 
     @property
     def path(self):
+        if self._path:
+            return self._path
         return path.join(self.repository.directory,"profiles", self.profile)
 
+    @path.setter
+    def path(self, value):
+        self._path = value
+        return value
+
     @classmethod
     def from_string(cls, repository, s):
-        parts = filter(None, s.split())
+        parts = [x for x in s.split() if x]
         if len(parts) == 3 and parts[0] in cls.available_arch:
             return Profile(repository, parts[1], parts[0])
         return None
@@ -170,7 +178,7 @@ class Profile(object):
                                                  self.repository.directory)
 
 
-class ProfileRepository(object):
+class ProfileRepository():
     """
     Репозиторий либо скачивается, либо берется из кэша
     """
@@ -209,7 +217,7 @@ class ProfileRepository(object):
         if git.is_private_url(url):
             try:
                 makeDirectory(rpath)
-                os.chmod(rpath, 0700)
+                os.chmod(rpath, 0o700)
             except OSError:
                 pass
         git.cloneRepository(url, rpath, branch)
@@ -261,8 +269,13 @@ class ProfileRepository(object):
         Получить список профилей репозитория
         """
         profiles_desc = path.join(self.directory, "profiles/profiles.desc")
-        return filter(None, (Profile.from_string(self, line)
+        return list(filter(None, (Profile.from_string(self, line)
-                             for line in readLinesFile(profiles_desc)))
+                             for line in readLinesFile(profiles_desc))))
+    @staticmethod
+    def get_local_profiles(directory):
+        profiles_desc = path.join(directory, "profiles/profiles.desc")
+        return list(filter(None, (Profile.from_string(Profile, line)
+                                  for line in readLinesFile(profiles_desc))))
 
     def __repr__(self):
         return "<ProfileRepository %s url=%s>" % (self.directory, self.url)

pym/update/update_info.py

@@ -16,7 +16,6 @@
 
 import os
 from os import path
-from itertools import ifilter
 from calculate.core.datavars import DataVarsCore
 from calculate.core.server.gen_pid import search_worked_process
 from calculate.lib.cl_template import SystemIni
@@ -24,7 +23,7 @@ from calculate.lib.utils.content import getCfgFiles
 from calculate.lib.utils.files import getRunCommands, readFile, writeFile
 
 
-class UpdateInfo(object):
+class UpdateInfo():
     """
     Информационный объект о процессе обновления
     """
@@ -71,7 +70,7 @@ class UpdateInfo(object):
         """
         Проверить есть ли уже запущенная копия console-gui
         """
-        return any(ifilter(lambda x: "cl-console-gui" in x, getRunCommands()))
+        return any([x for x in getRunCommands() if "cl-console-gui" in x])
 
     def update_already_run(self):
         """

pym/update/update_tasks.py

@@ -26,3 +26,4 @@ class EmergeMark:
     RevdepRebuild = "revdep rebuild"
     Prelink = "prelink"
     Automagic = "check for auto depends"
+    SaveTag = "save latest calculate update tag"

pym/update/utils/cl_setup_update.py

@@ -18,7 +18,7 @@ import sys
 from calculate.core.server.func import Action
 from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
 from calculate.lib.utils.files import FilesError
-from calculate.update.update import UpdateError
+from ..update import UpdateError
 from calculate.lib.utils.git import GitError
 
 _ = lambda x: x

pym/update/utils/cl_update.py

@@ -15,24 +15,251 @@
 #  limitations under the License.
 
 import sys
-from calculate.core.server.func import Action, Tasks
+from calculate.core.server.func import Action, Tasks, AllTasks
 from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
 from calculate.lib.cl_template import TemplatesError
 from calculate.lib.utils.binhosts import BinhostError
 from calculate.lib.utils.files import FilesError, readFile
-from calculate.update.update import UpdateError
+from ..update import UpdateError
-from calculate.update.emerge_parser import EmergeError
+from ..emerge_parser import EmergeError
 from calculate.lib.utils.git import GitError
-from calculate.lib.utils.portage import (EmergeLog,
+from calculate.lib.utils.portage import (EmergeLog, isPkgInstalled,
                                          EmergeLogNamedTask, PackageList)
-from calculate.update.update_tasks import EmergeMark
+from ..update_tasks import EmergeMark
 
 _ = lambda x: x
 setLocalTranslate('cl_update3', sys.modules[__name__])
 __ = getLazyLocalTranslate(_)
 
 
-class UpdateConditions(object):
+def get_synchronization_tasks(object_name):
+    Object = lambda s: "%s.%s"%(object_name, s)
+    return [
+        
+        {'name': 'reps_synchronization',
+         'group': __("Repositories synchronization"),
+         'tasks': [
+             # создать объект проверки PGP
+             {'name': 'prepare_gpg',
+              'method': Object("prepare_gpg()"),
+             },
+             # создать объект хранилище серверов обновлений
+             {'name': 'create_binhost_data',
+              'method': Object('create_binhost_data()')
+             },
+             {
+              'name': 'perform_migration_system_pre_check',
+              'method': Object('perform_migration_system_pre_check()'),
+              'condition': lambda Get, GetBool: (Get('cl_update_ignore_level')  == 'off')
+             },
+             # проверить валиден ли текущий хост
+             {'name': 'check_current_binhost',
+              'message': __("Checking current binhost"),
+              'essential': False,
+              'method': Object('check_current_binhost(update.cl_update_binhost)'),
+              'condition': lambda GetBool, Get: (
+                    not GetBool('update.cl_update_binhost_recheck_set') and
+                    not Get('update.cl_update_binhost_choice') and
+                    Get('update.cl_update_binhost_choice') != 'auto' and
+                    Get('update.cl_update_sync_rep') and
+                    Get('update.cl_update_binhost') and 
+                    Get('update.cl_update_use_migration_host') == 'off')
+             },
+             {'name': 'drop_packeges_cache',
+              'method': 'Update.drop_packages_cache()',
+              'depend': AllTasks.failed_all("check_current_binhost")
+              },
+             {'name': 'not_use_search:failed_base_binhost',
+              'error': __("Failed to use base binhost"),
+              'method': Object("delete_binhost()"),
+              'depend': AllTasks.failed_all("check_current_binhost")
+             },
+             {'name': 'check_chosen_binhost',
+              'message': _("Checking chosen binhost"),
+              'method': Object('check_chosen_binhost(update.cl_update_binhost_choice)'),
+              'essential': False,
+              'condition': lambda GetBool, Get: (
+                  not GetBool('update.cl_update_binhost_recheck_set') and
+                  Get('update.cl_update_binhost_choice') and
+                  Get('update.cl_update_binhost_choice') != 'auto'),
+              'depend': AllTasks.hasnot("check_current_binhost")},
+             {'name': 'failed_check_chosen_binhost',
+              'method': Object('check_if_migration()'),
+              'depend': AllTasks.failed_all("check_chosen_binhost")},
+             {'name': 'group_find_binhost',
+              'group': '',
+              'while': (~AllTasks.has_any("detect_best_binhost") &
+                         ((AllTasks.failed_all("update_packages_cache")
+                           & ~AllTasks.has_any("not_use_search")) |
+                          ~AllTasks.has_any("sync_reps"))) & Tasks.success(),
+              'condition': lambda GetBool, Get: (GetBool('update.cl_update_usetag_set') and
+                                                 Get('update.cl_update_sync_rep')),
+              'tasks': [
+                 # найти лучший сервер обновлений
+                 {'name': 'detect_best_binhost',
+                  'method': Object(f'detect_best_binhost({object_name.lower()}.cl_update_ignore_level)'),
+                  'essential': False,
+                  'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search") &
+                             (AllTasks.failed_one_of("check_current_binhost") |
+                              AllTasks.failed_one_of("check_chosen_binhost") |
+                              (AllTasks.hasnot("check_current_binhost") &
+                               AllTasks.hasnot("check_chosen_binhost")) |
+                             AllTasks.success_all("sync_reps"))),
+                  'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'off',
+                 },
+                 {'name': 'interrupt_on_no_leveled_binhost',
+                  'method': Object("interrupt_on_no_leveled_binhost()"),
+                  'essential': True,
+                  'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search") &
+                             (~AllTasks.success_one_of("check_current_binhost")) &
+                             AllTasks.failed_all("detect_best_binhost")),
+                  'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'off'
+                 },
+                 {'name': 'set_migration_binhost',
+                  'method': Object('set_migration_host()'),
+                  'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search")|
+                             AllTasks.success_all("sync_reps")),
+                  'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'on'
+                 },
+                 # запасная синхронизация, в ходе которой ветки обновляются до
+                 # master
+                #  {'name': 'sync_reps_fallback',
+                #   'foreach': 'update.cl_update_sync_rep',
+                #   'message':
+                #       __("Fallback syncing the {eachvar:capitalize} repository"),
+                #   'method': Object('syncRepositories(eachvar,True)'),
+                #   'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"),
+                #  },
+                #  # обновление переменных информации из binhost
+                #  {'name': 'sync_reps_fallback:update_binhost_list',
+                #   'method': Object('update_binhost_list()'),
+                #   'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"),
+                #  },
+                #  # найти лучший сервер обновлений 
+                #  {'name': 'sync_reps_fallback:detect_best_binhost',
+                #   'method': Object('detect_best_binhost()'),
+                #   'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"),
+                #  },
+                 {'name': 'sync_reps',
+                  'foreach': 'update.cl_update_sync_rep',
+                  'message': __("Checking {eachvar:capitalize} updates"),
+                  'method': Object('syncRepositories(eachvar)'),
+                  'condition': lambda Get: Get('update.cl_update_sync_rep'),
+                  'depend': Tasks.success() & ~AllTasks.success_all("update_packages_cache")
+                 },
+                 {'name': 'sync_reps:update_local_info_binhost',
+                  'method': Object('update_local_info_binhost()'),
+                  'condition': lambda Get: Get("update.cl_update_binhost_choice") == 'auto'
+                                           or not Get("update.cl_update_binhost_choice")
+                                           or Get("update.cl_update_binhost_recheck_set"),
+                  'depend': AllTasks.hasnot('check_chosen_binhost')
+                 },
+                 {'name': 'sync_reps:update_binhost_list',
+                  'essential': False,
+                  'method': Object('update_binhost_list()'),
+                  'condition': lambda GetBool: GetBool('update.cl_update_outdate_set')
+                 },
+                 {'name': 'sync_reps:update_packages_cache',
+                  'message': __("Update packages index"),
+                  'method': Object('download_packages(update.cl_update_portage_binhost,'
+                            'update.cl_update_package_cache, update.cl_update_package_cache_sign,'
+                            'update.cl_update_gpg)'),
+                  'condition': lambda Get, GetBool: (
+                      Get('update.cl_update_package_cache') and (
+                          Get('update.cl_update_outdate_set') == 'on' or
+                          Get('update.cl_update_package_cache_set') == 'on'))
+                 },
+                ],
+             },
+             {'name': 'no_server',
+              'error': __("Failed to find the binary updates server"),
+              'method': Object("delete_binhost()"),
+              # method: который должен удалить текущую информацию о сервере обновлений
+              'depend': (~Tasks.has_any("failed_base_binhost") & (Tasks.failed() |
+                         Tasks.success() & AllTasks.failed_one_of("update_packages_cache"))),
+              'condition': lambda GetBool, Get: (GetBool('update.cl_update_usetag_set') and
+                                                 Get('update.cl_update_sync_rep')),
+             },
+             {'name': 'sync_reps',
+              'foreach': 'update.cl_update_sync_rep',
+              'message': __("Checking {eachvar:capitalize} updates"),
+              'method': Object('syncRepositories(eachvar)'),
+              'condition': lambda Get, GetBool: (Get('update.cl_update_sync_rep') and
+                                                 not GetBool('update.cl_update_usetag_set')),
+             },
+             {'name': 'update_rep_list',
+              'message': __("Repository cache update"),
+              'method': Object('update_rep_list()'),
+              'condition': lambda Get: (isPkgInstalled(
+                  "app-eselect/eselect-repository", prefix=Get('cl_chroot_path')) and
+                  Get('cl_chroot_path') != "/"),
+              'essential': False,
+             },
+             {'name': 'sync_other_reps',
+              'foreach': 'update.cl_update_other_rep_name',
+              'message': __("Syncing the {eachvar:capitalize} repository"),
+              'method': Object('syncOtherRepository(eachvar)'),
+              'condition': lambda GetBool: (GetBool('update.cl_update_other_set') or
+                                            not GetBool('update.cl_update_other_git_exists'))
+             },
+             {'name': 'trim_reps',
+              'foreach': 'update.cl_update_sync_rep',
+              'message': __("Cleaning the history of the "
+                            "{eachvar:capitalize} repository"),
+              'method': Object('trimRepositories(eachvar)'),
+              'condition': lambda Get: (Get('update.cl_update_sync_rep') and
+                                        Get('update.cl_update_onedepth_set') == 'on')
+              },
+             {'name': 'sync_reps:regen_cache',
+              'foreach': 'update.cl_update_sync_overlay_rep',
+              'essential': False,
+              'method': Object('regenCache(eachvar)'),
+              'condition': (
+                  lambda Get: (Get('update.cl_update_outdate_set') == 'on' and
+                               Get('update.cl_update_egencache_force') != 'skip' or
+                               Get('update.cl_update_egencache_force') == 'force'))
+             },
+             {'name': 'sync_other_reps:regen_other_cache',
+              'foreach': 'update.cl_update_other_rep_name',
+              'method': Object('regenCache(eachvar)'),
+              'essential': False,
+             },
+             {'name': 'eix_update',
+              'message': __("Updating the eix cache for "
+                            "{update.cl_update_eix_repositories}"),
+              'method': Object('eixUpdate(cl_repository_name)'),
+              'condition': (
+                  lambda Get: (Get('update.cl_update_outdate_set') == 'on' and
+                               Get('update.cl_update_eixupdate_force') != 'skip' or
+                               Get('update.cl_update_eixupdate_force') == 'force'))
+             },
+             {'name': 'update_setup_cache',
+              'message': __("Updating the cache of configurable packages"),
+              'method': Object('updateSetupCache()'),
+              'essential': False,
+              'condition': lambda Get: Get('update.cl_update_outdate_set') == 'on'
+             },
+             {'name': 'sync_reps:cleanpkg',
+              'message': __("Removing obsolete distfiles and binary packages"),
+              'method': Object('cleanpkg()'),
+              'condition': (
+                  lambda Get: Get('update.cl_update_cleanpkg_set') == 'on' and
+                              Get('update.cl_update_outdate_set') == 'on'),
+              'essential': False
+             },
+             # сообщение удачного завершения при обновлении репозиториев
+             {'name': 'success_syncrep',
+              'message': __("Synchronization finished"),
+              'depend': (Tasks.success() & Tasks.has_any("sync_reps",
+                                                         "sync_other_reps",
+                                                         "emerge_metadata",
+                                                         "eix_update")),
+              }
+         ]
+         },
+    ]
+
+class UpdateConditions():
     @staticmethod
     def was_installed(pkg, task_name):
         def func():
@@ -46,7 +273,8 @@ class UpdateConditions(object):
             task = EmergeLog(EmergeLogNamedTask(task_name))
             return (bool(PackageList(task.list)[pkg])
                     or Get('cl_update_force_depclean_set') == 'on'
-                    or Get('cl_update_outdated_kernel_set') == 'on')
+                    or Get('cl_update_outdated_kernel_set') == 'on'
+                    or Get('cl_update_world_hash_set') == 'on')
         return func
 
     @staticmethod
@@ -57,6 +285,13 @@ class UpdateConditions(object):
             return False
         else:
             return True
+    
+    @staticmethod
+    def check_world_updated_after_tag_save():
+        def func():
+            task = EmergeLog(EmergeLogNamedTask(EmergeMark.SaveTag))
+            return task.did_update_world_happen()
+        return func
 
 class ClUpdateAction(Action):
     """
@@ -76,6 +311,13 @@ class ClUpdateAction(Action):
          'method': 'Update.save_with_bdeps()',
          'essential': False
          },
+        {'name': 'update_fastlogin_domain',
+         'method': "Update.update_fastlogin_domain_path()"
+        },
+        {'name': 'drop_portage_hash_on_sync',
+         'method': 'Update.drop_portage_state_hash()',
+         'condition': lambda Get: Get('cl_update_outdate_set') == 'on'
+         },
         {'name': 'premerge_group',
          'group': __("Checking for updates"),
          'tasks': [
@@ -84,12 +326,25 @@ class ClUpdateAction(Action):
              'method': 'Update.premerge("-uDN","@world")',
              'condition': lambda Get: (
                  Get('cl_update_sync_only_set') == 'off' and
-                 Get('cl_update_pretend_set') == 'on')
+                 Get('cl_update_pretend_set') == 'on') and \
+                   (Get('cl_update_world') != "update" or
+                    Get('cl_update_outdate_set') == 'on' or
+                    Get('cl_update_settings_changes_set') == 'on' or
+                    Get('cl_update_binhost_recheck_set') == 'on' or
+                    Get('cl_update_force_fix_set') == 'on' or
+                    Get('update.cl_update_package_cache_set') == 'on')
             }],
          },
         {'name': 'update',
-         'condition': lambda Get:Get('cl_update_pretend_set') == 'off',
+         'condition': lambda Get:Get('cl_update_pretend_set') == 'off' and \
-         },
+                           (Get('cl_update_world') != "update" or
+                            Get('cl_update_outdate_set') == 'on' or
+                            Get('cl_update_settings_changes_set') == 'on' or
+                            Get('cl_update_binhost_recheck_set') == 'on' or
+                            Get('cl_update_force_fix_set') == 'on' or
+                            Get('cl_update_available_set') == 'on' or
+                            Get('update.cl_update_package_cache_set') == 'on')
+             },
         {'name': 'update_other',
          'condition': lambda Get: ( Get('cl_update_pretend_set') == 'off' and
                                     Get('cl_update_sync_only_set') == 'off')
@@ -103,10 +358,12 @@ class ClUpdateAction(Action):
                         '"-uDN","@world")',
               }
          ],
-         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
+         'condition': lambda Get: (Get('cl_update_sync_only_set') == 'off' and
+                                   (Get('update.cl_update_outdate_set') == 'on' or
+                                    Get('cl_update_settings_changes_set') == 'on'))
          },
         {'name': 'update_other:update_perl',
-         'group': __("Updating Perl"),
+         'group': __("Updating Perl modules"),
          'tasks': [
              {'name': 'update_other:perl_cleaner',
               'message': __('Find & rebuild packages and Perl header files '
@@ -137,7 +394,7 @@ class ClUpdateAction(Action):
               'message': __('Updating Kernel modules'),
               'method': 'Update.emerge("","@module-rebuild")',
               'condition': UpdateConditions.was_installed(
-                  'sys-kernel/.*source', EmergeMark.KernelModules),
+                  'sys-kernel/(calculate-sources|gentoo-kernel|vanilla-kernel)', EmergeMark.KernelModules),
               'decoration': 'Update.update_task("%s")' %
                             EmergeMark.KernelModules
              },
@@ -162,7 +419,7 @@ class ClUpdateAction(Action):
               'message': __('Checking reverse dependencies'),
               'method': 'Update.revdep_rebuild("revdep-rebuild")',
               'condition': lambda Get: (Get(
-                  'cl_update_skip_rb_set') == 'off' and
+                  'cl_update_revdep_rebuild_set') == 'on' and
                   UpdateConditions.was_installed(
                       '.*', EmergeMark.RevdepRebuild)()),
               'decoration': 'Update.update_task("%s")' %
@@ -176,8 +433,10 @@ class ClUpdateAction(Action):
              },
          ]
         },
-        {'name': 'update:set_upto_date_cache',
+        {'name': 'set_upto_date_cache',
-         'method': 'Update.setUpToDateCache()'
+         'method': 'Update.setUpToDateCache()',
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off' and \
+                                  Get('cl_update_pretend_set') == 'off'
         }
     ]
 
@@ -192,146 +451,47 @@ class ClUpdateAction(Action):
         {'name': 'check_run',
          'method': 'Update.checkRun(cl_update_wait_another_set)'
         },
-        {'name': 'reps_synchronization',
+        {'name': 'check_if_world_was_updated',
-         'group': __("Repositories synchronization"),
+         'method': 'Update.update_set_current_saved_tag()',
-         'tasks': [
+         'essential': False,
-             # запасная синхронизация, в ходе которой ветки обновляются до
+         'condition': lambda Get: (Get('cl_update_sync_only_set') == 'off' and \
-             # master
+                Get('cl_update_pretend_set') == 'off' and \
-             {'name': 'sync_reps_fallback',
+                UpdateConditions.check_world_updated_after_tag_save()())
-              'foreach': 'cl_update_sync_rep',
+        },
-              'message':
+        ] + get_synchronization_tasks("Update") + [
-                  __("Fallback syncing the {eachvar:capitalize} repository"),
+        {'name': 'system_configuration',
-              'method': 'Update.syncRepositories(eachvar,True)',
-              'condition': lambda Get,GetBool: (
-                    GetBool('cl_update_usetag_set') and
-                    not Get('cl_update_binhost_data')[0])
-             },
-             # обновление переменных информации из binhost
-             {'name': 'update_binhost_list',
-              'method': 'Update.update_binhost_list()',
-              'condition': lambda Get,GetBool: (
-                    GetBool('cl_update_usetag_set') and
-                    not Get('cl_update_binhost_data')[0])
-             },
-             {'name': 'sync_reps',
-              'foreach': 'cl_update_sync_rep',
-              'message': __("Checking {eachvar:capitalize} updates"),
-              'method': 'Update.syncRepositories(eachvar)',
-              'condition': lambda Get: Get('cl_update_sync_rep')
-             },
-             {'name': 'check_binhost',
-              'method': 'Update.check_binhost()',
-              'condition': lambda GetBool: GetBool('cl_update_usetag_set')
-             },
-             {'name': 'sync_other_reps',
-              'foreach': 'cl_update_other_rep_name',
-              'message': __("Syncing the {eachvar:capitalize} repository"),
-              'method': 'Update.syncLaymanRepository(eachvar)',
-              'condition': lambda Get: Get('cl_update_other_set') == 'on'
-             },
-             {'name': 'trim_reps',
-              'foreach': 'cl_update_sync_rep',
-              'message': __("Cleaning the history of the "
-                            "{eachvar:capitalize} repository"),
-              'method': 'Update.trimRepositories(eachvar)',
-              'condition': lambda Get: (Get('cl_update_sync_rep') and
-                                        Get('cl_update_onedepth_set') == 'on')
-              },
-             {'name': 'sync_reps:regen_cache',
-              'foreach': 'cl_update_sync_overlay_rep',
-              'essential': False,
-              'method': 'Update.regenCache(eachvar)',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_egencache_force') != 'skip' or
-                               Get('cl_update_egencache_force') == 'force'))
-             },
-             {'name': 'sync_other_reps:regen_other_cache',
-              'foreach': 'cl_update_other_rep_name',
-              'method': 'Update.regenCache(eachvar)',
-              'essential': False,
-             },
-             {'name': 'emerge_metadata',
-              'message': __("Metadata transfer"),
-              'method': 'Update.emergeMetadata()',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_metadata_force') != 'skip' or
-                               Get('cl_update_metadata_force') == 'force'))
-             },
-             {'name': 'eix_update',
-              'message': __("Updating the eix cache for "
-                            "{cl_update_eix_repositories}"),
-              'method': 'Update.eixUpdate(cl_repository_name)',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_eixupdate_force') != 'skip' or
-                               Get('cl_update_eixupdate_force') == 'force'))
-             },
-             {'name': 'update_setup_cache',
-              'message': __("Updating the cache of configurable packages"),
-              'method': 'Update.updateSetupCache()',
-              'essential': False,
-              'condition': lambda Get: Get('cl_update_outdate_set') == 'on'
-             },
-             {'name': 'sync_reps:cleanpkg',
-              'message': __("Removing obsolete distfiles and binary packages"),
-              'method': 'Update.cleanpkg()',
-              'condition': (
-                  lambda Get: Get('cl_update_cleanpkg_set') == 'on' and
-                              Get('cl_update_outdate_set') == 'on'),
-              'essential': False
-             },
-             {'name': 'update_packages_cache',
-              'message': __("Update packages index"),
-              'method': 'Update.download_packages(cl_update_portage_binhost,'
-                        'cl_update_package_cache)',
-              'essential': False,
-              'condition': lambda Get, GetBool: (
-                  GetBool('cl_update_usetag_set') and
-                  Get('cl_update_package_cache') and (
-                      Get('cl_update_outdate_set') == 'on' or
-                      Get('cl_update_package_cache_set') == 'on'))
-              },
-             # сообщение удачного завершения при обновлении репозиториев
-             {'name': 'success_syncrep',
-              'message': __("Synchronization finished"),
-              'depend': (Tasks.success() & Tasks.has_any("sync_reps",
-                                                         "sync_other_reps",
-                                                         "emerge_metadata",
-                                                         "eix_update")),
-              }
-         ]
-         },
-        {'name': 'reps_synchronization',
          'group': __("System configuration"),
          'tasks': [
+            {'name': 'binhost_changed',
+             'method': 'Update.message_binhost_changed()'
+             },
             {'name': 'revision',
              'message': __("Fixing the settings"),
              'method': 'Update.applyTemplates(install.cl_source,'
                        'cl_template_clt_set,True,None,False)',
-             'condition': lambda Get: (Get('cl_templates_locate') and
+             'condition': lambda Get, GetBool: (Get('cl_templates_locate') and
                                        (Get('cl_update_world') != "update" or
-                                        Get('cl_update_outdate_set') == 'on' or
+                                        GetBool('cl_update_outdate_set') or
-                                        Get('cl_update_binhost_recheck_set') == 'on' or
+                                        GetBool('cl_update_binhost_recheck_set') or
-                                        Get('cl_update_force_fix_set') == 'on'))
+                                        GetBool('cl_update_force_fix_set') or
-
+                                        GetBool('update.cl_update_package_cache_set')))
              },
             {'name': 'dispatch_conf',
              'message': __("Updating configuration files"),
              'method': 'Update.dispatchConf()',
-             'condition': lambda Get: (Get('cl_dispatch_conf') != 'skip' and
+             'condition': lambda Get, GetBool: (Get('cl_dispatch_conf') != 'skip' and
                                        Get('cl_update_pretend_set') == 'off' and
-                                       (Get('cl_update_binhost_recheck_set') == 'on' or
+                                       (GetBool('cl_update_outdate_set') or
-                                        Get('cl_update_outdate_set') == 'on' or
+                                        GetBool('cl_update_binhost_recheck_set') or
-                                        Get('cl_update_force_fix_set') == 'on'))
+                                        GetBool('cl_update_force_fix_set') or
-             },
+                                        GetBool('update.cl_update_package_cache_set')))
-            {'name': 'binhost_changed',
-             'method': 'Update.message_binhost_changed()'
              },
          ]
          }
     ] + emerge_tasks + [
+        {'name': 'check success configure',
+         'method': 'Update.check_success_configure()',
+         },
         {'name': 'failed',
          'error': __("Update failed"),
          'depend': (Tasks.failed() & Tasks.hasnot("interrupt") &
@@ -341,10 +501,56 @@ class ClUpdateAction(Action):
          'depend': Tasks.failed_all("check_schedule")
          },
         # сообщение удачного завершения при обновлении ревизии
+
+        {'name': 'drop_portage_hash',
+         'method': 'Update.drop_portage_state_hash()',
+         'depend': Tasks.failed()
+         },
+        {'name': 'drop_world_hash',
+         'method': 'Update.drop_world_state_hash()',
+         'depend': Tasks.failed()},
+         {'name': 'update:set_current_level',
+         'method': 'Update.update_increment_current_level()',
+         'depend': (Tasks.success() & Tasks.hasnot("interrupt") &
+             Tasks.success_all("update") & Tasks.hasnot("check_schedule")), 
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off' and
+                Get('cl_update_pretend_set') == 'off' and
+                Get('update.cl_update_use_migration_host') == 'on'
+         },
+         {'name': 'update_world:set_latest_tag',
+         'method': 'Update.update_set_current_saved_tag()',
+         'depend': (Tasks.success() & Tasks.hasnot("interrupt") &
+             Tasks.success_all("update") & Tasks.hasnot("check_schedule")), 
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off' and
+                Get('cl_update_pretend_set') == 'off',
+         'decoration': 'Update.update_task("%s")' % EmergeMark.SaveTag
+         },
+        {'name': 'update:save_portage_hash',
+         'method': 'Update.save_portage_state_hash()',
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
+         },
+        {'name': 'update:save_world_hash',
+         'method': 'Update.save_world_state_hash()',
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
+         },
+        {'name': 'clear_migration_host',
+         'method': 'Update.delete_binhost()',
+         'depend': (Tasks.hasnot("check_schedule")), 
+         'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'on'
+         },
+        # сообщение удачного завершения при обновлении ревизии
         {'name': 'success_rev',
          'message': __("System update finished!"),
          'condition': lambda Get: (Get('cl_update_rev_set') == 'on' and
-                                    Get('cl_update_pretend_set') == 'off')
+                                    Get('cl_update_pretend_set') == 'off' and
+                                    Get('cl_update_sync_only_set') == 'off')
+         },
+        # сообщение удачного завершения при обновлении ревизии
+        {'name': 'success_sync_only',
+         'message': __("System sync finished!"),
+         'condition': lambda Get: (Get('cl_update_rev_set') == 'on' and
+                                    Get('cl_update_pretend_set') == 'off' and
+                                    Get('cl_update_sync_only_set') == 'on')
          },
         # сообщение удачного завершения при пересоздании world
         {'name': 'success_world',

pym/update/utils/cl_update_profile.py

@@ -20,7 +20,7 @@ from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
 from calculate.lib.cl_template import TemplatesError
 from calculate.lib.utils.binhosts import BinhostError
 from calculate.lib.utils.files import FilesError
-from calculate.update.update import UpdateError
+from ..update import UpdateError
 from calculate.lib.utils.git import GitError
 
 _ = lambda x: x
@@ -47,73 +47,32 @@ class ClUpdateProfileAction(Action):
          'method': 'Update.migrateCacheRepository('
                    'cl_update_profile_url,cl_update_profile_branch,'
                    'cl_update_profile_storage)',
-         'message': __("Repository transfer"),
+         'condition': lambda Get: Get('cl_update_profile_url')},
-         'condition': lambda Get: not (
+        {'name': 'profile_from_url',
-             Get('cl_update_profile_storage').is_local(
+         'group': __('setting up from url'),
-                 Get('cl_update_profile_url'),
-                 Get('cl_update_profile_branch')))
-         },
-        {'name': 'reconfigure_vars1',
-         'method': 'Update.invalidateVariables("cl_update_profile_storage")',
-         'depend': Tasks.has('migrate_repository')
-         },
-        {'name': 'check_datavars',
-         'error': _("Profile not found in master"),
-         'condition': lambda Get: not Get('update.cl_update_profile_datavars')
-         },
-        {'name': 'drop_binhosts',
-         'method': 'Update.drop_binhosts(update.cl_update_profile_datavars)'
-         },
-        {'name': 'reconfigure_vars',
-         'method': 'Update.reconfigureProfileVars(cl_update_profile_datavars,'
-                          'cl_chroot_path)'
-         },
-        {'name': 'reps_synchronization',
-         'group': __("Repositories synchronization"),
          'tasks': [
-             {'name': 'sync_reps',
+            {'message': __("Repository transfer"),
-              'foreach': 'cl_update_profile_sync_rep',
+             'condition': lambda Get: not (
-              'message': __("Syncing the {eachvar:capitalize} repository"),
+                 Get('cl_update_profile_storage').is_local(
-              'method': 'Update.syncRepositories(eachvar)',
+                     Get('cl_update_profile_url'),
-              # TODO: неиспользуемое условие
+                     Get('cl_update_profile_branch'))) and Get('cl_update_profile_check_sync_allowed'),
-              # 'condition': lambda Get: Get('cl_update_profile_sync_rep')
+             },
-              },
+            {'name': 'reconfigure_vars1',
-             {'name': 'sync_reps:regen_cache',
+             'method': 'Update.invalidateVariables("cl_update_profile_storage")',
-              'foreach': 'cl_update_sync_overlay_rep',
+             },
-              'message': __("Updating the {eachvar:capitalize} repository cache"),
+            {'name': 'check_datavars',
-              'essential': False,
+             'error': _("Profile not found in master"),
-              'method': 'Update.regenCache(eachvar)',
+             'condition': lambda Get: not Get('update.cl_update_profile_datavars'),
-              'condition': (
+             },
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
+            {'name': 'drop_binhosts',
-                               Get('cl_update_metadata_force') != 'skip' or
+             'method': 'Update.drop_binhosts(update.cl_update_profile_datavars)',
-                               Get('cl_update_metadata_force') == 'force'))
+             },
-              },
+            {'name': 'reconfigure_vars',
-             {'name': 'emerge_metadata',
+             'method': 'Update.reconfigureProfileVars(cl_update_profile_datavars,'
-              'message': __("Metadata transfer"),
+                              'cl_chroot_path)',
-              'method': 'Update.emergeMetadata()',
+             },
-              'condition': (
+         ],
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
+         'depend': Tasks.has('migrate_repository')
-                               Get('cl_update_metadata_force') != 'skip' or
-                               Get('cl_update_metadata_force') == 'force'))
-              },
-             {'name': 'eix_update',
-              'message': __("Updating the eix cache for "
-                            "{cl_update_eix_repositories}"),
-              'method': 'Update.eixUpdate(cl_repository_name)',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_eixupdate_force') != 'skip' or
-                               Get('cl_update_eixupdate_force') == 'force'))
-              },
-             # сообщение удачного завершения при обновлении репозиториев
-             {'name': 'success_syncrep',
-              'message': __("Synchronization finished"),
-              'depend': (Tasks.success() & Tasks.has_any("sync_reps",
-                                                         "sync_other_reps",
-                                                         "emerge_metadata",
-                                                         "eix_update")),
-              }
-         ]
          },
         {'name': 'reps_synchronization',
          'group': __("Setting up the profile"),
@@ -122,6 +81,9 @@ class ClUpdateProfileAction(Action):
               'message': __("Switching to profile {cl_update_profile_system}"),
               'method': 'Update.setProfile(cl_update_profile_system)'
               },
+             {'name': 'rename_custom',
+              'method': 'Update.rename_custom_files()',
+             },
              {'name': 'revision',
               'message': __("Fixing the settings"),
               'method': 'Update.applyProfileTemplates(cl_template_clt_set,'

pym/update/variables/__init__.py

@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-import action
+from . import action
-import update
+from . import update
 
 section = "update"

pym/update/variables/action.py

@@ -27,6 +27,7 @@ class VariableAcUpdateSync(ReadonlyVariable):
     """
     def get(self):
         action = self.Get("cl_action")
-        if action in ("sync", 'update_profile'):
+        if action in ("sync", 'update_profile') and \
+            not self.Get('cl_merge_pkg'):
             return "on"
         return "off"

pym/update/wsdl_update.py

@@ -20,17 +20,18 @@ from calculate.lib.datavars import VariableError,DataVarsError
 
 from calculate.core.server.func import WsdlBase
 from calculate.install.install import InstallError
-from calculate.update.update import Update, UpdateError
+from .update import Update, UpdateError
 from calculate.lib.utils.git import GitError
-from utils.cl_update import ClUpdateAction
+from .utils.cl_update import ClUpdateAction
-from utils.cl_update_profile import ClUpdateProfileAction
+from .utils.cl_update_profile import ClUpdateProfileAction
-from utils.cl_setup_update import ClSetupUpdateAction
+from .utils.cl_setup_update import ClSetupUpdateAction
 from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _
 
 setLocalTranslate('cl_update3', sys.modules[__name__])
 __ = getLazyLocalTranslate(_)
 
 
+
 class Wsdl(WsdlBase):
     methods = [
         #
@@ -67,27 +68,32 @@ class Wsdl(WsdlBase):
                                     normal=(
                                         'cl_update_binhost_stable_opt_set',
                                         'cl_update_binhost_recheck_set',
-                                        'cl_update_with_bdeps_opt_set'
+                                        'cl_update_with_bdeps_opt_set',
+                                        'cl_update_binhost_choice',
+                                        'cl_update_rep_hosting_choice',
                                     ),
                                     expert=(
-                                    'cl_update_sync_only_set',
+                                        'cl_update_other_set',
-                                    'cl_update_other_set',
+                                        'cl_update_sync_only_set',
-                                    'cl_update_pretend_set',
+                                        'cl_update_pretend_set',
-                                    'cl_update_sync_rep',
+                                        'cl_update_sync_rep',
-                                    'cl_update_emergelist_set',
+                                        'cl_update_emergelist_set',
-                                    'cl_update_check_rep_set',
+                                        'cl_update_check_rep_set',
-                                    'cl_update_force_fix_set',
+                                        'cl_update_force_fix_set',
-                                    'cl_update_world',
+                                        'cl_update_ignore_level',
-                                    'cl_update_egencache_force',
+                                        'cl_update_force_level',
-                                    'cl_update_eixupdate_force',
+                                        'cl_update_world',
-                                    'cl_update_skip_rb_set',
+                                        'cl_update_gpg_force',
-                                    'cl_update_wait_another_set',
+                                        'cl_update_egencache_force',
-                                    'cl_update_autocheck_schedule_set',
+                                        'cl_update_eixupdate_force',
-                                    'cl_update_onedepth_set',
+                                        'cl_update_revdep_rebuild_set',
-                                    'cl_update_cleanpkg_set',
+                                        'cl_update_wait_another_set',
-                                    'cl_update_branch_data',
+                                        'cl_update_autocheck_schedule_set',
-                                    'cl_templates_locate',
+                                        'cl_update_onedepth_set',
-                                    'cl_verbose_set', 'cl_dispatch_conf'),
+                                        'cl_update_cleanpkg_set',
+                                        'cl_update_branch_data',
+                                        'cl_templates_locate',
+                                        'cl_verbose_set', 'cl_dispatch_conf'),
                                     next_label=_("Run"))]},
         #
         # Сменить профиль
@@ -118,7 +124,7 @@ class Wsdl(WsdlBase):
             'native_error': (VariableError, DataVarsError,
                              InstallError, UpdateError, GitError),
             # значения по умолчанию для переменных этого метода
-            'setvars': {'cl_action!': 'update_profile'},
+            'setvars': {'cl_action!': 'update_profile', 'cl_update_world_default': "rebuild"},
             # описание груп (список лямбда функций)
             'groups': [
                 lambda group: group(_("Repository"),

setup.py

@@ -18,7 +18,7 @@
 #  limitations under the License.
 
 __app__ = "calculate-update"
-__version__ = "3.2.2"
+__version__ = "3.7.3" #TODO bump version
 
 import os
 from glob import glob
@@ -67,7 +67,7 @@ class install_data(module_install_data.install_data):
                             os.chmod(out,chmod)
                         self.outfiles.append(out)
 
-data_files = [('/usr/libexec/calculate', [('data/cl-git-wrapper', 0755)])]
+data_files = [('/usr/libexec/calculate', [('data/cl-git-wrapper', 0o755)])]
 
 packages = [
     "calculate."+str('.'.join(root.split(os.sep)[1:]))