TG-653

правка получения переменной

pym/update/emerge_parser.py

@@ -88,7 +88,10 @@ class CommandExecutor():
 
     def close(self):
         if self.child is not None:
-            self.child.close()
+            try:
+                self.child.close()
+            except pexpect.ExceptionPexpect:
+                self.child.close(force=True)
             self.child = None
 
     def success(self):
@@ -594,7 +597,6 @@ class InstallingPackage(NotifierInformationBlock):
     binary = None
 
     token = ">>> Installing "
-    #TODO \033 - unicode symbol? probably gonna have to fix this
     re_block = re.compile(
         "Installing \({c}(\d+){c} "
         "of {c}(\d+){c}\) {c}([^\s\033]+){c}".format(c=_color_block))
@@ -640,7 +642,6 @@ class RevdepPercentBlock(NotifierInformationBlock):
     token = "Collecting system binaries"
     end_token = [re.compile("Assigning files to packages|"
                             "All prepared. Starting rebuild")]
-    # re_block = re.compile("\[\s(\d+)%\s\]")
     re_block = re.compile("\[\s(\d+)%\s\]")
     action = None
 

pym/update/profile.py

@@ -151,11 +151,19 @@ class Profile():
         self.repository = repository
         self.profile = profile
         self.arch = arch
+        self._path = None
 
     @property
     def path(self):
+        if self._path:
+            return self._path
         return path.join(self.repository.directory,"profiles", self.profile)
 
+    @path.setter
+    def path(self, value):
+        self._path = value
+        return value
+
     @classmethod
     def from_string(cls, repository, s):
         parts = [x for x in s.split() if x]
@@ -263,6 +271,11 @@ class ProfileRepository():
         profiles_desc = path.join(self.directory, "profiles/profiles.desc")
         return list(filter(None, (Profile.from_string(self, line)
                              for line in readLinesFile(profiles_desc))))
+    @staticmethod
+    def get_local_profiles(directory):
+        profiles_desc = path.join(directory, "profiles/profiles.desc")
+        return list(filter(None, (Profile.from_string(Profile, line)
+                                  for line in readLinesFile(profiles_desc))))
 
     def __repr__(self):
         return "<ProfileRepository %s url=%s>" % (self.directory, self.url)

pym/update/update.py

@@ -15,6 +15,7 @@
 #  limitations under the License.
 from functools import wraps
 import random
+import asyncio
 
 import sys
 from os import path
@@ -29,6 +30,8 @@ from calculate.lib.datavars import DataVarsError, VariableError, Variable
 from calculate.lib.utils.tools import AddonError
 from calculate.lib.utils.colortext.palette import TextState
 from calculate.lib.utils.colortext import get_color_print
+
+from calculate.core.server.core_interfaces import RerunTrigger
 from .emerge_parser import RevdepPercentBlock
 from .datavars import DataVarsUpdate
 from .update_info import UpdateInfo
@@ -64,7 +67,7 @@ from calculate.lib.utils.files import (getProgPath, STDOUT, removeDir,
                                        PercentProgress, process, getRunCommands,
                                        readFile, listDirectory, pathJoin,
                                        find, FindFileType,quite_unlink,
-                                       writeFile, makeDirectory)
+                                       writeFile, makeDirectory, clearDirectory)
 from . import emerge_parser
 import logging
 from .emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
@@ -230,6 +233,8 @@ class Update(MethodsInterface):
                         status = git.getStatusInfo(rpath)
                         if not status or status['files']:
                             need_update = True
+
+
                 except GitError as e:
                     need_update = True
                 if need_update:
@@ -246,7 +251,6 @@ class Update(MethodsInterface):
                         info_outdated = True
             if info_outdated:
                 self.raiseOutdate()
-                dv.Set('cl_update_outdate_set', 'on', force=True)
         finally:
             self.unstash_cache(rpath, name)
         # TODO: debug1
@@ -255,12 +259,21 @@ class Update(MethodsInterface):
 
     def raiseOutdate(self):
         self.clVars.Set('cl_update_outdate_set', 'on', force=True)
+        if self.clVars.Get('cl_update_other_set') == 'auto':
+            self.clVars.Set('cl_update_other_set', 'on')
 
     def setAutocheckParams(self, status, interval, update_other, cleanpkg):
         """
         Настроить параметры автопроверки обновлений
         """
-        onoff = lambda x: "on" if x else "off"
+        def onoff(val):
+            if val and val not in ['off', 'auto']:
+                return 'on'
+            elif val == 'auto':
+                return 'auto'
+            else:
+                return 'off'
+
         self.clVars.Write('cl_update_autocheck_set', onoff(status), True)
         self.clVars.Write('cl_update_autocheck_interval', interval, True)
         self.clVars.Write('cl_update_other_set', onoff(update_other), True)
@@ -373,6 +386,32 @@ class Update(MethodsInterface):
                 self.endTask(True)
         return True
 
+
+    def checkUrl(self, host, host_urls, repname, dv, git):
+        url, rpath, revision = (
+            dv.Select([host_urls, "cl_update_rep_path",
+                       "cl_update_rep_rev"],
+                      where="cl_update_rep_name", eq=repname, limit=1))
+        swap = {'main': 'master', 'master': 'main'}
+        if not git.checkUrl(url):
+            self.printWARNING(_("Git %s is unavailable, scaning next host") % url)
+            return None, url, rpath, revision
+        if revision in swap:
+            ini = SystemIni(dv)
+            cached_main_branch = ini.getVar("branch", repname)
+            revision = cached_main_branch if cached_main_branch else "main"
+            try:
+                dv.Get("cl_update_profile_storage").get_profiles(url, revision)
+                ini.setVar("branch", {repname: revision})
+                return host, url, rpath, revision
+            except GitError:
+                revision = swap[revision]
+                ini.setVar("branch", {repname: revision})
+        return host, url, rpath, revision
+
+
+
+
     @variable_module("update")
     def syncRepositories(self, repname, fallback_sync=False,
                          clean_on_error=True):
@@ -381,17 +420,55 @@ class Update(MethodsInterface):
         """
         dv = self.clVars
         check_status = dv.GetBool('update.cl_update_check_rep_set')
-        url, rpath, revision = (
-            dv.Select(["cl_update_rep_url", "cl_update_rep_path",
-                       "cl_update_rep_rev"],
-                      where="cl_update_rep_name", eq=repname, limit=1))
-
-        if not url or not rpath:
-            raise UpdateError(_("Configuration variables for repositories "
-                                "are not setup"))
         git = self.getGit()
-        if not git.checkUrl(url):
-            raise UpdateError(_("Git %s is unavailable") % url)
+        saved_host = dv.Get('cl_update_rep_hosting') if 'cl_update_rep_hosting' in dv.allVars else ''
+        working_host = ''
+        # проверка необходимости синхронизации других оверлеев
+        if 'cl_update_rep_list' in dv.allVars and dv.Get('cl_update_rep_list'):
+            chosen_hosting = dv.Get("cl_update_rep_hosting_choice")
+            if chosen_hosting:
+                for host_urls, host in dv.Get('cl_update_inner_rep_list'):
+                    if host == chosen_hosting:
+                        working_host, url, rpath, revision = self.checkUrl(host, host_urls, repname, dv, git)
+                        if working_host:
+                            break
+            elif saved_host:
+                for host_urls, host in dv.Get('cl_update_inner_rep_list'):
+                    if host == saved_host:
+                        working_host, url, rpath, revision = self.checkUrl(host, host_urls, repname, dv, git)
+                        if working_host:
+                            break
+            # Берем первый доступный хостинг из списка
+            if not working_host:
+                for host_urls, host in dv.Get('cl_update_inner_rep_list'):
+                    if chosen_hosting == host:
+                        continue
+                    working_host, url, rpath, revision = self.checkUrl(host, host_urls, repname, dv, git)
+                    if working_host:
+                        break
+            if not working_host:
+                dv.Write('cl_update_rep_hosting', ' ')
+                raise UpdateError(_("Git is unavailable"))
+            if working_host:
+                dv.Write('cl_update_rep_hosting', working_host)
+
+        #--------------DEPRECATED--------------#
+        else:
+            url, rpath, revision = (
+                dv.Select(["cl_update_rep_url", "cl_update_rep_path",
+                           "cl_update_rep_rev"],
+                          where="cl_update_rep_name", eq=repname, limit=1))
+
+            if not url or not rpath:
+                raise UpdateError(_("Configuration variables for repositories "
+                                    "are not setup"))
+            git = self.getGit()
+            if not git.checkUrl(url):
+                raise UpdateError(_("Git %s is unavailable") % url)
+        # --------------DEPRECATED--------------#
+        # check for readonly fs
+        if not os.access(rpath, os.W_OK):
+            return True
         chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
         if chroot_path == '/':
             rpath_orig = rpath
@@ -405,15 +482,13 @@ class Update(MethodsInterface):
         try:
             if clean_on_error:
                 try:
-                    repconf = ReposConf(dv.Get('cl_update_reposconf'),
-                        dv.Get('cl_update_reposconf_dir'),
-                        prefix=chroot_path)
-                    repconf.add(repname, url, rpath_orig)
                     if not self._syncRepository(repname, url, rpath, revision,
                                                 cb_progress=self.setProgress,
                                                 clean=check_status,
                                                 notask=fallback_sync):
+                        self.compare_repo_cache(repname, url, rpath_orig, chroot_path)
                         return "skip"
+                    self.compare_repo_cache(repname, url, rpath_orig, chroot_path)
                     return True
                 except GitError as e:
                     if not isinstance(e, NotGitError):
@@ -431,8 +506,18 @@ class Update(MethodsInterface):
                                              cb_progress=self.setProgress,
                                              clean=check_status,
                                              notask=fallback_sync)
-                        removeDir(rpath)
-                        shutil.move(rpath_new, rpath)
+                        if not isinstance(e, NotGitError):
+                            removeDir(rpath)    
+                            shutil.move(rpath_new, rpath)
+                        else:
+                            #delete everything in rpath dir, but not rpath itself
+                            for root, dirs, files in os.walk(rpath):
+                                for f in files:
+                                    os.unlink(os.path.join(root, f))
+                                for d in dirs:
+                                    shutil.rmtree(os.path.join(root, d))
+                            for i in os.listdir(rpath_new):
+                                shutil.move(os.path.join(rpath_new, i), rpath)
                     except OSError as e:
                         raise UpdateError(_("Failed to modify the "
                                             "{repname} repository").format(
@@ -455,6 +540,24 @@ class Update(MethodsInterface):
 
     metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
 
+    def compare_repo_cache(self, repname, url, rpath_orig, chroot_path):
+        if not self.clVars.GetBool('cl_update_settings_changes_set'):
+            repconf = ReposConf(self.clVars.Get('cl_update_reposconf'),
+                                self.clVars.Get('cl_update_reposconf_dir'),
+                                prefix=chroot_path)
+            repconf.add(repname, url, rpath_orig)
+            ini = SystemIni(self.clVars)
+            ps = PortageState()
+            new_portage_state_hash = ps.get_state()
+            ini.setVar('system', {'portage_hash': new_portage_state_hash})
+            self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True)
+        else:
+            repconf = ReposConf(self.clVars.Get('cl_update_reposconf'),
+                                self.clVars.Get('cl_update_reposconf_dir'),
+                                prefix=chroot_path)
+            repconf.add(repname, url, rpath_orig)
+
+
     def stash_cache(self, rpath, name):
         """
         Спрятать кэш
@@ -520,6 +623,8 @@ class Update(MethodsInterface):
                                    limit=1)
         repdirname = path.basename(rpath)
         self.stash_cache(rpath, repdirname)
+        rpath_old = f"{rpath}_old"
+        makeDirectory(rpath_old)
         try:
             if Git.is_git(rpath):
                 self.addProgress()
@@ -527,13 +632,27 @@ class Update(MethodsInterface):
                 for perc in p.progress():
                     self.setProgress(perc)
             else:
+                for root, dirs, files in os.walk(rpath):
+                    for f in files:
+                        os.replace(os.path.join(rpath, f), os.path.join(rpath_old, f))
+                    for d in dirs:
+                        shutil.move(os.path.join(rpath, d), os.path.join(rpath_old, d))
+                clearDirectory(rpath)
                 p = process(emerge, "--sync", repname, stderr=STDOUT)
             if p.failed():
+                for root, dirs, files in os.walk(rpath_old):
+                    for f in files:
+                        os.replace(os.path.join(rpath_old, f), os.path.join(rpath, f))
+                    for d in dirs:
+                        shutil.move(os.path.join(rpath_old, d), os.path.join(rpath, d))
                 raise UpdateError(
                     _("Failed to update the {rname} repository").format(
                         rname=repname),
                     addon=p.read())
+            else:
+                self.clVars.Set('cl_update_outdate_set', Variable.On, force=True)
         finally:
+            removeDir(rpath_old)
             self.unstash_cache(rpath, repdirname)
         return True
 
@@ -546,12 +665,17 @@ class Update(MethodsInterface):
         Обновить кэш метаданных репозитория
         """
         egenCache = self.get_prog_path('/usr/bin/egencache')
+        chroot_path = self.clVars.Get('builder.cl_builder_path')
         if not egenCache:
             raise UpdateError(_("The Portage tool is not found"))
+        repconf = ReposConf(self.clVars.Get('cl_update_reposconf'),
+                            self.clVars.Get('cl_update_reposconf_dir'),
+                            chroot_path)
+        repos_path = {x[0]: x[1] for x in list(repconf.get_all_base_repositories()) + list(repconf.get_other_repositories())}
         if repname in self.clVars.Get('cl_update_rep_name'):
-            path_rep = self.clVars.Select('cl_update_rep_path',
-                                          where='cl_update_rep_name',
-                                          eq=repname, limit=1)
+            path_rep = repos_path.get(repname)
+            if not path_rep:
+                return True
             repo_name = readFile(
                 path.join(path_rep, "profiles/repo_name")).strip()
             if repo_name != repname:
@@ -928,7 +1052,7 @@ class Update(MethodsInterface):
         self.set_need_update(False)
         return True
 
-    def _startEmerging(self, emerge):
+    def _startEmerging(self, emerge, start_msg=None):
         """
         Настроить и выполнить emerge
         """
@@ -939,6 +1063,8 @@ class Update(MethodsInterface):
                     self.update_map["{CATEGORY}/{PF}".format(**pkg)] = \
                         rv.partition(":")[0]
         emerge.command.send("yes\n")
+        if start_msg is not None and isinstance(start_msg, str):
+            self.startTask(start_msg)
         emerge.emerging.add_observer(self._printEmergePackage)
         emerge.installing.add_observer(self._printInstallPackage)
         emerge.uninstalling.add_observer(self._printUninstallPackage)
@@ -1026,7 +1152,7 @@ class Update(MethodsInterface):
                 self._display_install_package(emerge, emergelike=True)
                 self._display_error(emerge.prepare_error)
                 raise
-            self._startEmerging(emerge)
+            self._startEmerging(emerge, _("Fetching..."))
         return True
 
     def depclean(self):
@@ -1125,6 +1251,8 @@ class Update(MethodsInterface):
                     _("Failed to use the new profile. Try again."))
             for var_name in ('cl_update_rep_path',
                              'cl_update_rep_url',
+                             'cl_update_rep_list',
+                             'cl_update_binhost_set',
                              'cl_update_rep_name',
                              'cl_update_branch',
                              'cl_update_binhost_list',
@@ -1294,7 +1422,7 @@ class Update(MethodsInterface):
                                 "appropriate updates"))
         else:
             with writeFile(self.get_bin_cache_filename()) as f:
-                f.write(datas[0].strip()+"\n")
+                f.write(datas[0].strip() + "\n")
         if write_binhost:
             if hosts[0] != self.clVars.Get('update.cl_update_binhost'):
                 self.refresh_binhost = True
@@ -1355,28 +1483,31 @@ class Update(MethodsInterface):
         return True
 
     def delete_binhost(self):
-        self.clVars.Delete('cl_update_binhost', location="system")
-        try:
-            bin_cache_fn = self.get_bin_cache_filename()
-            if path.exists(bin_cache_fn):
-                os.unlink(bin_cache_fn)
-        except OSError:
-            raise UpdateError(
-                _("Failed to remove cached ini.env of binary repository"))
-        try:
-            for varname in ('update.cl_update_package_cache',
-                            'update.cl_update_package_cache_sign'):
-                fn = self.clVars.Get(varname)
-                if path.exists(fn):
-                    os.unlink(fn)
-        except OSError:
-            raise UpdateError(
-                _("Failed to remove cached Package index"))
-        # удалить binhost
-        binhost_fn = self.inchroot(
-            self.clVars.Get("update.cl_update_portage_binhost_path"))
-        if path.exists(binhost_fn):
-            os.unlink(binhost_fn)
+        if not self.clVars.GetBool('update.cl_update_binhost_set') or self.clVars.GetBool('cl_update_use_migration_host'):
+            self.clVars.Write('cl_update_binhost', ' ', location="system")
+            try:
+                bin_cache_fn = self.get_bin_cache_filename()
+                if path.exists(bin_cache_fn):
+                    os.unlink(bin_cache_fn)
+            except OSError:
+                raise UpdateError(
+                    _("Failed to remove cached ini.env of binary repository"))
+            try:
+                for varname in ('update.cl_update_package_cache',
+                                'update.cl_update_package_cache_sign'):
+                    fn = self.clVars.Get(varname)
+                    if path.exists(fn):
+                        os.unlink(fn)
+            except OSError:
+                raise UpdateError(
+                    _("Failed to remove cached Package index"))
+            # удалить binhost
+            binhost_fn = self.inchroot(
+                self.clVars.Get("update.cl_update_portage_binhost_path"))
+            if path.exists(binhost_fn):
+                os.unlink(binhost_fn)
+        if self.clVars.GetBool('update.cl_update_binhost_set') and self.clVars.GetBool('cl_update_use_migration_host'):
+            self.clVars.Write('cl_update_binhost', self.clVars.Get('cl_update_save_cur_binhost') , location="system")
         return False
 
     def update_binhost_list(self, dv=None):
@@ -1454,6 +1585,9 @@ class Update(MethodsInterface):
 
     def download_packages(self, url_binhost, packages_fn, packages_sign_fn, gpg):
         quite_unlink(packages_fn)
+        packages_ini = os.path.join(packages_sign_fn.split("/Packages.as")[0], 'ini.env')
+        quite_unlink(packages_ini)
+        orig_ini = Binhosts.packages_check_ini(url_binhost.split("/grp")[0]).decode('utf-8')
         orig_packages = Binhosts.fetch_packages(url_binhost)
         try:
             with writeFile(packages_fn) as f:
@@ -1461,6 +1595,8 @@ class Update(MethodsInterface):
                 pi["TTL"] = str(30 * DAYS)
                 pi["DOWNLOAD_TIMESTAMP"] = str(int(time.time()))
                 pi.write(f)
+            with open(packages_ini, 'w') as f:
+                f.write(orig_ini)
         except (OSError, IOError):
             raise UpdateError(_("Failed to save Packages"))
         self.endTask(True)
@@ -1476,11 +1612,7 @@ class Update(MethodsInterface):
                 f.write(Binhosts.fetch_packages_sign(url_binhost).decode("UTF-8"))
         except BinhostSignError:
             for fn in (packages_fn, packages_sign_fn):
-                if path.exists(fn):
-                    try:
-                        os.unlink(fn)
-                    except OSError:
-                        pass
+                quite_unlink(fn)
             self.clVars.Set("update.cl_update_bad_sign_set", Variable.On)
             self.clVars.Set('update.cl_update_binhost_recheck_set', Variable.On)
             self.clVars.Set('cl_update_package_cache_set', Variable.Off, force=True)
@@ -1498,9 +1630,7 @@ class Update(MethodsInterface):
         BadEnv = 7
         EnvNotFound = 8
         SkipSlower = 9
-        UnknownError = 10
-        
-
+        UnknownError = 10   
 
         @staticmethod
         def humanReadable(reason):
@@ -1515,8 +1645,7 @@ class Update(MethodsInterface):
                 Update.Reason.Skip: "SKIP",
                 Update.Reason.SkipSlower: "",
                 Update.Reason.Success: "",
-                #TODO formulate better
-                Update.Reason.LevelWrong: "FAILED (binhost level is wrong)"
+                Update.Reason.LevelWrong: "FAILED (binhost level is low)"
             }.get(reason,reason)
 
     def _get_binhost_logger(self):
@@ -1533,10 +1662,33 @@ class Update(MethodsInterface):
     def create_binhost_data(self):
         dv = self.clVars
         last_ts = dv.Get('cl_update_last_timestamp')
-        if dv.GetBool('cl_update_binhost_stable_opt_set'):
-            binhost_list = dv.Get('cl_update_binhost_list')
+        if dv.Get('update.cl_update_binhost_choice') == 'auto' \
+                    or (not dv.Get('update.cl_update_binhost') and not dv.Get('update.cl_update_binhost_choice'))\
+                or dv.GetBool('update.cl_update_binhost_recheck_set'):
+            dv.Write('cl_update_binhost_set', "off")
+            dv.Set('cl_update_binhost_set', 'off')
+
+        if dv.GetBool('update.cl_update_binhost_recheck_set'):
+            if dv.Get('cl_update_binhost_stable_opt_set'):
+                binhost_list = dv.Get('cl_update_binhost_list')
+            else:
+                binhost_list = dv.Get('cl_update_binhost_unstable_list')
+        elif dv.Get('update.cl_update_binhost_choice') and dv.Get('update.cl_update_binhost_choice') != 'auto':
+            binhost = dv.Get('cl_update_binhost_choice')
+            if not binhost.startswith('https://') and not binhost.startswith('ftp://') and not binhost.startswith(
+                    'http://'):
+                binhost = f"https://{binhost}"
+            binhost_list = [binhost]
+        elif dv.Get('update.cl_update_binhost') and dv.GetBool('update.cl_update_binhost_set') \
+                and dv.Get('update.cl_update_binhost_choice') != 'auto':
+            binhost = dv.Get('cl_update_binhost')
+            binhost_list = [binhost]
         else:
-            binhost_list = dv.Get('cl_update_binhost_unstable_list')
+            if dv.Get('cl_update_binhost_stable_opt_set'):
+                binhost_list = dv.Get('cl_update_binhost_list')
+            else:
+                binhost_list = dv.Get('cl_update_binhost_unstable_list')
+
         self.binhosts_data = Binhosts(
             # значение малозначимо, поэтому берётся из собирающей системы
             dv.GetInteger('cl_update_binhost_timeout'),
@@ -1545,6 +1697,7 @@ class Update(MethodsInterface):
             last_ts, binhost_list,
             self.get_arch_machine(),
             gpg=dv.Get('update.cl_update_gpg'))
+
         return True
 
     @variable_module("update")
@@ -1558,8 +1711,7 @@ class Update(MethodsInterface):
         retval = []
         skip_check_status = False
         actual_reason = self.Reason.UnknownError
-        # highest_level_found = int(self.clVars.Get('update.cl_update_level'))
-        for binhost in sorted(binhosts_data.get_binhosts(), reverse=True):
+        for binhost in sorted(asyncio.run(binhosts_data.get_asyncbinhosts()), reverse=True):
             host = binhost.host
             if not binhost.valid:
                 reason = self.Reason.WrongBinhost 
@@ -1579,14 +1731,11 @@ class Update(MethodsInterface):
 
                 else:
                     # SUCCESS
-                    if not binhost.downgraded or stabilization:
-                        if not ignore_level and not self.is_binhost_level_valid(binhost):
-                            reason = self.Reason.LevelWrong
-                            # if highest_level_found < binhost.level:
-                            #     highest_level_found = binhost.level
-                        else:
-                            host = "-> %s" % host
-                            reason = self.Reason.Success
+                    if not ignore_level and self.compare_update_level(binhost.level) < 0:
+                        reason = self.Reason.LevelWrong
+                    elif not binhost.downgraded or stabilization:
+                        host = "-> %s" % host
+                        reason = self.Reason.Success
                     else:
                         reason = self.Reason.Skip
             elif binhost.downgraded:
@@ -1607,43 +1756,60 @@ class Update(MethodsInterface):
                 timestamp=binhost.timestamp,
                 reason=Update.Reason.humanReadable(reason)))
         if not retval:
-            if actual_reason is self.Reason.BadSign:
+            if actual_reason is self.Reason.LevelWrong:
+                raise UpdateError(_("Failed to find trusted server with current level"))
+            elif actual_reason is self.Reason.BadSign:
                 raise UpdateError(_("Failed to find the reliable server with appropriate updates"))
             elif actual_reason in (self.Reason.Outdated,
                                    self.Reason.Skip,
                                    self.Reason.Updating):
                 raise UpdateError(_("Failed to find the server with appropriate updates"))
-            elif actual_reason is self.Reason.LevelWrong:
-                # self.clVars.Set('cl_update_max_level_found', highest_level_found)#TODO change to write
-                self.set_migration_mode(True)
-                #TODO add translation
-                raise UpdateError(_("Failed to find the server with appropriate level of updates"))
             else:
                 raise UpdateError(_("Failed to find the working server with updates"))
         return retval
     
     def is_binhost_level_valid(self, binhost):
-        return self.compare_update_level(binhost) == 0
+        return self.compare_update_level(binhost.level) == 0
 
     # @variable_module("update")
-    def compare_update_level(self, binhost):
+    def compare_update_level(self, level):
         # < 0  binhost level low
         # == 0 binhost level adequate
         # > 0  binhost level high
-        if binhost.level is None:
+        local_lvl = self.clVars.Get("update.cl_update_level")
+        if level is None or local_lvl == "":
             return None
-        return binhost.level - int(self.clVars.Get("update.cl_update_level"))
+        return level - int(local_lvl)
 
-    def check_current_binhost(self, binhost_url):
+    def is_branch_set(self):
+        #basically, if any of the branches are specified, we won't use migration
+        return any(x != "tag" for x in self.clVars.Get("cl_update_branch_name"))
+
+    def is_level_forced(self):
+        flvl = self.clVars.Get("cl_update_force_level")
+        return flvl.isdigit() and int(flvl) > 0
+
+    def perform_migration_system_pre_check(self):
+        branch_is_set_check = self.is_branch_set()
+        if branch_is_set_check:
+            self.clVars.Set("cl_update_ignore_level", Variable.On)
+        else:
+            if self.is_level_forced():
+                self.clVars.Set("cl_update_level", int(self.clVars.Get("cl_update_force_level")) - 1)
+                self.set_migration_mode(True)
+        return True
+
+    def check_current_binhost(self, binhost_url, rewrite_binhost=True):
         """
         Проверка текущего сервера обновлений на валидность
         """
-        if not binhost_url in self.binhosts_data.binhost_list:
-            raise UpdateError(_("Current binhost is absent in list of update servers"))
+        # if not binhost_url in self.binhosts_data.binhost_list:
+        #     raise UpdateError(_("Current binhost is absent in list of update servers"))
         binhost = self.binhosts_data.get_binhost(binhost_url)
-
+        ignore_level = self.clVars.Get("cl_update_ignore_level") == Variable.On
+        cur_level_valid = self.is_current_level_a_digit() if not ignore_level else None
         if binhost.valid and not binhost.outdated and not binhost.downgraded \
-                and self.is_binhost_level_valid(binhost):
+                and (ignore_level or not cur_level_valid or self.is_binhost_level_valid(binhost)):
             if binhost.status == self.binhosts_data.BinhostStatus.Success:
                 self.clVars.Set('update.cl_update_binhost_data',
                         [[binhost.host, binhost.data,
@@ -1651,6 +1817,8 @@ class Update(MethodsInterface):
                          str(binhost.duration),
                          str(binhost.level)]],
                          force=True)
+                if not ignore_level and not cur_level_valid and not self.is_level_forced():
+                    self.pull_level_from_binhost(binhost)
                 self.endTask()
         else:
             if not binhost.valid:
@@ -1663,48 +1831,52 @@ class Update(MethodsInterface):
                 raise UpdateError(
                     _("Binary packages on the current binhost {} "
                       "are older than local").format(binhost_url))
-            elif self.check_current_level():
-                if not self.is_binhost_level_valid(binhost):
-                    # self.stash_binhost(binhost_url)
+            elif not ignore_level and not self.is_binhost_level_valid(binhost):
+                if binhost.level and self.compare_update_level(binhost.level) > 0:
                     self.set_migration_mode(True)
-                    #TODO translate
-                    raise UpdateError(
-                        _("Current binhost {} has a wrong update level. Current level: {} Binhost level: {}").\
-                                format(binhost_url, self.clVars.Get("update.cl_update_level"), binhost.level, ))
-            else:
-                self.pull_level_from_binhost(binhost)
+                raise UpdateError((
+                    _("Current binhost {} has wrong update level. Current level: {} Binhost level: {}").\
+                            format(binhost_url, self.clVars.Get("update.cl_update_level"), binhost.level)))
 
         if self.binhosts_data.gpg:
             packages_fn = self.clVars.Get('update.cl_update_package_cache')
             packages_sign_fn = self.clVars.Get('update.cl_update_package_cache_sign')
-            if path.exists(packages_fn) and path.exists(packages_sign_fn):
-                packages_sign = readFile(packages_sign_fn, binary=True)
-                pi = PackagesIndex(readFile(packages_fn))
-                pi.clean()
+            packages_ini = os.path.join(packages_sign_fn.split("/Packages.as")[0], 'ini.env')
+            if path.exists(packages_fn) and path.exists(packages_sign_fn) and path.exists(packages_ini):
+                local_ini = readFile(packages_ini, binary=True)
                 try:
-                    Binhosts.check_packages_signature(
-                        None, pi.get_value(), self.binhosts_data.gpg,
-                        sign=packages_sign)
-                    return True
-                except BinhostSignError:
-                    for fn in (packages_fn, packages_sign_fn):
-                        if path.exists(fn):
-                            try:
-                                os.unlink(fn)
-                            except OSError:
-                                pass
-            if binhost.bad_sign:
-                raise UpdateError(
-                    _("Current binhost {} has wrong signature").format(
-                        binhost_url))
+                    orig_ini = Binhosts.packages_check_ini(binhost.host)
+                    if not orig_ini == local_ini:
+                        for fn in (packages_fn, packages_sign_fn, packages_ini):
+                            quite_unlink(fn)
+                        self.clVars.Set('cl_update_package_cache_set', Variable.On)
+                except BinhostError:
+                    for fn in (packages_fn, packages_sign_fn, packages_ini):
+                        quite_unlink(fn)
+                    raise UpdateError(_("Failed to find ini.env on current binhost"))
+
+        if rewrite_binhost:
+            self.clVars.Write('cl_update_binhost', binhost_url)
         return True
 
+    def check_chosen_binhost(self, binhost):
+        if not binhost.startswith('https://') and not binhost.startswith('ftp://') and not binhost.startswith(
+                'http://'):
+            binhost = f"https://{binhost}"
+        res = self.check_current_binhost(binhost, rewrite_binhost=False)
+        if binhost != self.clVars.Get('update.cl_update_binhost'):
+            self.refresh_binhost = True
+        self.clVars.Write('cl_update_binhost_set', 'on')
+        self.clVars.Write('cl_update_binhost', binhost)
+        return True
+
+
     def get_migration_mirror_url(self, host, level):
         level_required = int(level) + 1
         return f"{host}level{level_required}"
 
     @variable_module("update")
-    def detect_best_binhost(self):
+    def detect_best_binhost(self, ignore_level):
         # выполняется переход с серверов unstable обновлении на stable
         # в этом случае не важно, что бинари могут старее текущих
         if (self.clVars.GetBool('cl_update_binhost_stable_opt_set') and
@@ -1712,31 +1884,72 @@ class Update(MethodsInterface):
             stabilization = True
         else:
             stabilization = False
-        current_level_is_valid = self.check_current_level()
+        #had to extract ignore level logic to cl_update because of variable module
+        current_level_is_valid = self.is_current_level_a_digit() if not ignore_level else False
         self.startTask(_("Searching new binhost"))
+
+        #add try catch 
+
         retval = self._search_best_binhost(self.binhosts_data,
-                 stabilization,
-                 ignore_level = not current_level_is_valid)
+                 stabilization, ignore_level or not current_level_is_valid)
 
         
         self.clVars.Set('update.cl_update_binhost_data',
             retval or Variable.EmptyTable, force=True)
-
-        if not current_level_is_valid:
-            level = int(self.clVars.Get("update.cl_update_binhost_level")[0])
-            self._set_update_level(level)
+        if not ignore_level:
+            best_binhost_level = int(self.clVars.Get("update.cl_update_binhost_level")[0])
+            #int(sorted(self.clVars.Get("update.cl_update_binhost_level"), key=int)[-1])
+            # int(self.clVars.Get("update.cl_update_binhost_level")[0])
+            
+            if not current_level_is_valid:
+                self._set_update_level(best_binhost_level)
+            else:
+                cmpr = self.compare_update_level(best_binhost_level)
+                if cmpr is None:
+                    self.printWARNING("Failed to find working binhost. As a temporary solution you can use "
+                                      "'cl-update --mirror-update https://mirror.calculate-linux.org/migrate' command")
+                elif cmpr > 0:
+                    self.set_migration_mode(True)
+                    raise UpdateError(_("Current level: {} Binhost level: {}").\
+                        format(self.clVars.Get("update.cl_update_level"), best_binhost_level))
+                elif cmpr < 0:
+                    self.delete_binhost()
+                    raise UpdateError(_("Failed to find binhost with level equal or higher than local level"))
         self.endTask()
         return True
 
+    #нужен так как detect_best_binhost не essential, ошибка не останавливает 
+    def interrupt_on_no_leveled_binhost(self):
+        # raise UpdateError(_("Failed to find binhost with level equal to or higher than local level"))
+        return False
+
     def set_migration_mode(self, val=True):
         val_to_write = Variable.On if val else Variable.Off
-        # self.clVars.Write("cl_update_use_migration_host", val_to_write, location="system")
-        self.clVars.Set("cl_update_use_migration_host", val_to_write)
+        self.clVars.Set("update.cl_update_use_migration_host", val_to_write)
         return True
 
 
+    def cmp_rep_tag_to_current_calc_rep_tag(self):
+        #TODO value to Var
+        repname = 'calculate'
+        revision = (self.clVars.Select(["cl_update_rep_rev"],
+                where="cl_update_rep_name", 
+                eq=repname, limit=1))[0]
+        saved_tag = self.get_current_saved_tag()
+        # git = self.getGit()
+        # rpath = '/var/db/repos/calculate'
+        # tggg = git.getCurrentTag(rpath)
+        if saved_tag is not None:
+            return int(revision) - saved_tag
+        else:
+            return 1
+
+
+    def get_current_saved_tag(self):
+        tg = self.clVars.Get("cl_update_saved_tag")
+        return int(tg) if tg else None
+
     def set_migration_host(self):
-        #TODO translate
         self.startTask(_("Setting up migration host"))
         dv = self.clVars
         last_ts = dv.Get('cl_update_last_timestamp')
@@ -1759,9 +1972,17 @@ class Update(MethodsInterface):
                     str(level)]]
         self.clVars.Set('update.cl_update_binhost_data',
             migr_binhost_data or Variable.EmptyTable, force=True)
+        self.clVars.Set('cl_update_binhost', migration_host.host)
         if not (migr_binhost_data):
-            #TODO translate
-            raise UpdateError(_("Failed to find the working migration server with appropriate level"))
+            raise UpdateError(_("Failed to find working migration host with current level"))
+        if not self.is_level_forced():
+            #tag_cmp = self.cmp_rep_tag_to_current_calc_rep_tag()
+            tag_cmp = 2
+            if tag_cmp <= 0:
+                self._update_increment_current_level()
+                self.delete_binhost()
+                self.set_migration_mode(False)
+                raise RerunTrigger(_("Raising level due to migration host being outdated"))
         self.endTask()
         return True
 
@@ -1844,6 +2065,30 @@ class Update(MethodsInterface):
         ini.delVar('system', 'portage_hash')
         return True
 
+    def save_world_state_hash(self):
+        ini = SystemIni(self.clVars)
+        ps = PortageState()
+        ps.paths = ['/var/lib/portage/world']
+        new_world_state_hash = ps.get_state()
+        ini.setVar('system', {'world_hash': new_world_state_hash})
+        return True
+
+    def drop_world_state_hash(self):
+        """
+        Сбросить состояние
+        """
+        ini = SystemIni(self.clVars)
+        ini.delVar('system', 'world_hash')
+        return True
+
+    def drop_packages_cache(self):
+        packages_fn = self.clVars.Get('update.cl_update_package_cache')
+        packages_sign_fn = self.clVars.Get('update.cl_update_package_cache_sign')
+        packages_ini = os.path.join(packages_sign_fn.split("/Packages.as")[0], 'ini.env')
+        for fn in (packages_fn, packages_sign_fn, packages_ini):
+            quite_unlink(fn)
+        return True
+
     def update_fastlogin_domain_path(self):
         try:
             if not self.clVars.Get('client.cl_remote_host'):
@@ -1864,51 +2109,77 @@ class Update(MethodsInterface):
         return True
 
     def update_increment_current_level(self):
-        #TODO add translation
-        self.startTask(_("Increasing update level"))
+        self.startTask(_("Increasing local update level"))
         self._update_increment_current_level()
-        self.set_migration_mode(False)
-        # self.unstash_binhost()
         self.delete_binhost()
+        self.set_migration_mode(False)
         self.endTask()
         return True
 
+    #should not do when on a branch
+    def update_set_current_saved_tag(self):
+        #TODO add to var?
+        repname = "calculate"
+        git = self.getGit()
+        rpath, revision = (
+                self.clVars.Select(["cl_update_rep_path",
+                "cl_update_rep_rev"],
+                where="cl_update_rep_name", 
+                eq=repname, limit=1))
+        try:
+            branch_name = self.clVars.Select(["cl_update_branch_name"],
+                    where="cl_update_branch_rep", eq=repname, limit=1)[0]
+            tag = git.getCurrentTag(rpath)
+            if branch_name == Git.Reference.Tag and tag.isdigit():
+                self._set_saved_tag(tag)
+            return True
+        except (ValueError, FileNotFoundError) as e:
+            return False
+
     def _update_increment_current_level(self):
         current_level = int(self.clVars.Get('cl_update_level'))
-        self.clVars.Write('cl_update_level', 
-                str(current_level + 1), location="system")
+        self._set_update_level(str(current_level + 1))
 
-    def check_current_level(self):
+    def is_current_level_a_digit(self):
         """
-        Возвращает True если есть значение в calculate.env
+        Возвращает True если значение update_level это цифра >=0
         """
-        return int(self.clVars.Get('cl_update_level')) >= 0
-            
+        # isdigit - True if it can be interp-ed as a number >=0
+        return self.clVars.Get('cl_update_level').isdigit()            
 
     def pull_level_from_binhost(self, binhost):
         self._set_update_level(binhost.level)
+        return True
 
     def _set_update_level(self, level):
-        if level:
-            self.clVars.Write('cl_update_level', 
-                    str(level), location="system")
-            return True
+        self._set_val_in_system(level, 'cl_update_level')
+    
+    def _set_saved_tag(self, tag):
+        self._set_val_in_system(tag, 'cl_update_saved_tag')
+
+    def _set_val_in_system(self, val, val_name):
+        """
+        val should be int >= 0
+        """
+        if isinstance(val, str):
+            val = val if val.isdigit() else 0 
+        elif isinstance(val, int):
+            val = val if val >= 0 else 0
         else:
-            self.clVars.Write('cl_update_level', 
-                    str(0), location="system")
-            return False
+            val = val if val else 0
+        self.clVars.Write(val_name, str(val), location="system")
 
-    # def stash_binhost(self, binhost_url):
-    #     """
-    #     Бекап адреса бинхоста
-    #     """
-    #     self.clVars.Write("cl_update_binhost_stash", binhost_url, location="system")
-
-    # def unstash_binhost(self):
-    #     """
-    #     Восстановление сохраненного бинхоста (если он есть)
-    #     """
-    #     val_to_write = self.clVars.Get("update.cl_update_binhost_stash")
-    #     if val_to_write:
-    #         self.clVars.Write('cl_update_binhost', val_to_write, location="system")
-    
+    def check_if_migration(self):
+        """
+        Проверка для переключения на миграционный хост при ошибке валидации выбранного бинхоста
+        """
+        if self.clVars.GetBool("update.cl_update_use_migration_host"):
+            self.clVars.Set('cl_update_save_cur_binhost', self.clVars.Get('cl_update_binhost'))
+            return True
+        raise UpdateError(_("Something goes wrong while validating chosen binhost"))
+
+    def check_success_configure(self):
+        try:
+            return self.clVars.GetBool("cl_success_configure")
+        except:
+            return True

pym/update/update_tasks.py

@@ -26,3 +26,4 @@ class EmergeMark:
     RevdepRebuild = "revdep rebuild"
     Prelink = "prelink"
     Automagic = "check for auto depends"
+    SaveTag = "save latest calculate update tag"

pym/update/utils/cl_update.py

@@ -47,6 +47,11 @@ def get_synchronization_tasks(object_name):
              {'name': 'create_binhost_data',
               'method': Object('create_binhost_data()')
              },
+             {
+              'name': 'perform_migration_system_pre_check',
+              'method': Object('perform_migration_system_pre_check()'),
+              'condition': lambda Get, GetBool: (Get('cl_update_ignore_level')  == 'off')
+             },
              # проверить валиден ли текущий хост
              {'name': 'check_current_binhost',
               'message': __("Checking current binhost"),
@@ -54,14 +59,33 @@ def get_synchronization_tasks(object_name):
               'method': Object('check_current_binhost(update.cl_update_binhost)'),
               'condition': lambda GetBool, Get: (
                     not GetBool('update.cl_update_binhost_recheck_set') and
+                    not Get('update.cl_update_binhost_choice') and
+                    Get('update.cl_update_binhost_choice') != 'auto' and
                     Get('update.cl_update_sync_rep') and
-                    Get('update.cl_update_binhost'))
+                    Get('update.cl_update_binhost') and 
+                    Get('update.cl_update_use_migration_host') == 'off')
              },
+             {'name': 'drop_packeges_cache',
+              'method': 'Update.drop_packages_cache()',
+              'depend': AllTasks.failed_all("check_current_binhost")
+              },
              {'name': 'not_use_search:failed_base_binhost',
               'error': __("Failed to use base binhost"),
               'method': Object("delete_binhost()"),
               'depend': AllTasks.failed_all("check_current_binhost")
              },
+             {'name': 'check_chosen_binhost',
+              'message': _("Checking chosen binhost"),
+              'method': Object('check_chosen_binhost(update.cl_update_binhost_choice)'),
+              'essential': False,
+              'condition': lambda GetBool, Get: (
+                  not GetBool('update.cl_update_binhost_recheck_set') and
+                  Get('update.cl_update_binhost_choice') and
+                  Get('update.cl_update_binhost_choice') != 'auto'),
+              'depend': AllTasks.hasnot("check_current_binhost")},
+             {'name': 'failed_check_chosen_binhost',
+              'method': Object('check_if_migration()'),
+              'depend': AllTasks.failed_all("check_chosen_binhost")},
              {'name': 'group_find_binhost',
               'group': '',
               'while': (~AllTasks.has_any("detect_best_binhost") &
@@ -73,11 +97,22 @@ def get_synchronization_tasks(object_name):
               'tasks': [
                  # найти лучший сервер обновлений
                  {'name': 'detect_best_binhost',
-                  'method': Object('detect_best_binhost()'),
+                  'method': Object(f'detect_best_binhost({object_name.lower()}.cl_update_ignore_level)'),
                   'essential': False,
                   'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search") &
-                             (~AllTasks.success_one_of("check_current_binhost") |
+                             (AllTasks.failed_one_of("check_current_binhost") |
+                              AllTasks.failed_one_of("check_chosen_binhost") |
+                              (AllTasks.hasnot("check_current_binhost") &
+                               AllTasks.hasnot("check_chosen_binhost")) |
                              AllTasks.success_all("sync_reps"))),
+                  'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'off',
+                 },
+                 {'name': 'interrupt_on_no_leveled_binhost',
+                  'method': Object("interrupt_on_no_leveled_binhost()"),
+                  'essential': True,
+                  'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search") &
+                             (~AllTasks.success_one_of("check_current_binhost")) &
+                             AllTasks.failed_all("detect_best_binhost")),
                   'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'off'
                  },
                  {'name': 'set_migration_binhost',
@@ -114,6 +149,10 @@ def get_synchronization_tasks(object_name):
                  },
                  {'name': 'sync_reps:update_local_info_binhost',
                   'method': Object('update_local_info_binhost()'),
+                  'condition': lambda Get: Get("update.cl_update_binhost_choice") == 'auto'
+                                           or not Get("update.cl_update_binhost_choice")
+                                           or Get("update.cl_update_binhost_recheck_set"),
+                  'depend': AllTasks.hasnot('check_chosen_binhost')
                  },
                  {'name': 'sync_reps:update_binhost_list',
                   'essential': False,
@@ -123,9 +162,8 @@ def get_synchronization_tasks(object_name):
                  {'name': 'sync_reps:update_packages_cache',
                   'message': __("Update packages index"),
                   'method': Object('download_packages(update.cl_update_portage_binhost,'
-                            'update.cl_update_package_cache,update.cl_update_package_cache_sign,'
+                            'update.cl_update_package_cache, update.cl_update_package_cache_sign,'
                             'update.cl_update_gpg)'),
-                  'essential': False,
                   'condition': lambda Get, GetBool: (
                       Get('update.cl_update_package_cache') and (
                           Get('update.cl_update_outdate_set') == 'on' or
@@ -161,7 +199,8 @@ def get_synchronization_tasks(object_name):
               'foreach': 'update.cl_update_other_rep_name',
               'message': __("Syncing the {eachvar:capitalize} repository"),
               'method': Object('syncOtherRepository(eachvar)'),
-              'condition': lambda GetBool: GetBool('update.cl_update_other_set')
+              'condition': lambda GetBool: (GetBool('update.cl_update_other_set') or
+                                            not GetBool('update.cl_update_other_git_exists'))
              },
              {'name': 'trim_reps',
               'foreach': 'update.cl_update_sync_rep',
@@ -234,7 +273,8 @@ class UpdateConditions():
             task = EmergeLog(EmergeLogNamedTask(task_name))
             return (bool(PackageList(task.list)[pkg])
                     or Get('cl_update_force_depclean_set') == 'on'
-                    or Get('cl_update_outdated_kernel_set') == 'on')
+                    or Get('cl_update_outdated_kernel_set') == 'on'
+                    or Get('cl_update_world_hash_set') == 'on')
         return func
 
     @staticmethod
@@ -245,6 +285,13 @@ class UpdateConditions():
             return False
         else:
             return True
+    
+    @staticmethod
+    def check_world_updated_after_tag_save():
+        def func():
+            task = EmergeLog(EmergeLogNamedTask(EmergeMark.SaveTag))
+            return task.did_update_world_happen()
+        return func
 
 class ClUpdateAction(Action):
     """
@@ -311,7 +358,9 @@ class ClUpdateAction(Action):
                         '"-uDN","@world")',
               }
          ],
-         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
+         'condition': lambda Get: (Get('cl_update_sync_only_set') == 'off' and
+                                   (Get('update.cl_update_outdate_set') == 'on' or
+                                    Get('cl_update_settings_changes_set') == 'on'))
          },
         {'name': 'update_other:update_perl',
          'group': __("Updating Perl modules"),
@@ -345,7 +394,7 @@ class ClUpdateAction(Action):
               'message': __('Updating Kernel modules'),
               'method': 'Update.emerge("","@module-rebuild")',
               'condition': UpdateConditions.was_installed(
-                  'sys-kernel/.*source', EmergeMark.KernelModules),
+                  'sys-kernel/(calculate-sources|gentoo-kernel|vanilla-kernel)', EmergeMark.KernelModules),
               'decoration': 'Update.update_task("%s")' %
                             EmergeMark.KernelModules
              },
@@ -402,6 +451,13 @@ class ClUpdateAction(Action):
         {'name': 'check_run',
          'method': 'Update.checkRun(cl_update_wait_another_set)'
         },
+        {'name': 'check_if_world_was_updated',
+         'method': 'Update.update_set_current_saved_tag()',
+         'essential': False,
+         'condition': lambda Get: (Get('cl_update_sync_only_set') == 'off' and \
+                Get('cl_update_pretend_set') == 'off' and \
+                UpdateConditions.check_world_updated_after_tag_save()())
+        },
         ] + get_synchronization_tasks("Update") + [
         {'name': 'system_configuration',
          'group': __("System configuration"),
@@ -433,6 +489,9 @@ class ClUpdateAction(Action):
          ]
          }
     ] + emerge_tasks + [
+        {'name': 'check success configure',
+         'method': 'Update.check_success_configure()',
+         },
         {'name': 'failed',
          'error': __("Update failed"),
          'depend': (Tasks.failed() & Tasks.hasnot("interrupt") &
@@ -442,27 +501,56 @@ class ClUpdateAction(Action):
          'depend': Tasks.failed_all("check_schedule")
          },
         # сообщение удачного завершения при обновлении ревизии
-        {'name': 'update:save_portage_hash',
-         'method': 'Update.save_portage_state_hash()',
-         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
-         },
+
         {'name': 'drop_portage_hash',
          'method': 'Update.drop_portage_state_hash()',
          'depend': Tasks.failed()
          },
-        #  #TODO 
+        {'name': 'drop_world_hash',
+         'method': 'Update.drop_world_state_hash()',
+         'depend': Tasks.failed()},
          {'name': 'update:set_current_level',
          'method': 'Update.update_increment_current_level()',
-         'depend': Tasks.success(), 
+         'depend': (Tasks.success() & Tasks.hasnot("interrupt") &
+             Tasks.success_all("update") & Tasks.hasnot("check_schedule")), 
          'condition': lambda Get: Get('cl_update_sync_only_set') == 'off' and
                 Get('cl_update_pretend_set') == 'off' and
                 Get('update.cl_update_use_migration_host') == 'on'
          },
+         {'name': 'update_world:set_latest_tag',
+         'method': 'Update.update_set_current_saved_tag()',
+         'depend': (Tasks.success() & Tasks.hasnot("interrupt") &
+             Tasks.success_all("update") & Tasks.hasnot("check_schedule")), 
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off' and
+                Get('cl_update_pretend_set') == 'off',
+         'decoration': 'Update.update_task("%s")' % EmergeMark.SaveTag
+         },
+        {'name': 'update:save_portage_hash',
+         'method': 'Update.save_portage_state_hash()',
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
+         },
+        {'name': 'update:save_world_hash',
+         'method': 'Update.save_world_state_hash()',
+         'condition': lambda Get: Get('cl_update_sync_only_set') == 'off'
+         },
+        {'name': 'clear_migration_host',
+         'method': 'Update.delete_binhost()',
+         'depend': (Tasks.hasnot("check_schedule")), 
+         'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'on'
+         },
         # сообщение удачного завершения при обновлении ревизии
         {'name': 'success_rev',
          'message': __("System update finished!"),
          'condition': lambda Get: (Get('cl_update_rev_set') == 'on' and
-                                    Get('cl_update_pretend_set') == 'off')
+                                    Get('cl_update_pretend_set') == 'off' and
+                                    Get('cl_update_sync_only_set') == 'off')
+         },
+        # сообщение удачного завершения при обновлении ревизии
+        {'name': 'success_sync_only',
+         'message': __("System sync finished!"),
+         'condition': lambda Get: (Get('cl_update_rev_set') == 'on' and
+                                    Get('cl_update_pretend_set') == 'off' and
+                                    Get('cl_update_sync_only_set') == 'on')
          },
         # сообщение удачного завершения при пересоздании world
         {'name': 'success_world',

pym/update/utils/cl_update_profile.py

@@ -47,71 +47,32 @@ class ClUpdateProfileAction(Action):
          'method': 'Update.migrateCacheRepository('
                    'cl_update_profile_url,cl_update_profile_branch,'
                    'cl_update_profile_storage)',
-         'message': __("Repository transfer"),
-         'condition': lambda Get: not (
-             Get('cl_update_profile_storage').is_local(
-                 Get('cl_update_profile_url'),
-                 Get('cl_update_profile_branch')))
-         },
-        {'name': 'reconfigure_vars1',
-         'method': 'Update.invalidateVariables("cl_update_profile_storage")',
-         'depend': Tasks.has('migrate_repository')
-         },
-        {'name': 'check_datavars',
-         'error': _("Profile not found in master"),
-         'condition': lambda Get: not Get('update.cl_update_profile_datavars')
-         },
-        {'name': 'drop_binhosts',
-         'method': 'Update.drop_binhosts(update.cl_update_profile_datavars)'
-         },
-        {'name': 'reconfigure_vars',
-         'method': 'Update.reconfigureProfileVars(cl_update_profile_datavars,'
-                          'cl_chroot_path)'
-         },
-        {'name': 'reps_synchronization',
-         'group': __("Repositories synchronization"),
+         'condition': lambda Get: Get('cl_update_profile_url')},
+        {'name': 'profile_from_url',
+         'group': __('setting up from url'),
          'tasks': [
-             {'name': 'sync_reps',
-              'foreach': 'cl_update_profile_sync_rep',
-              'message': __("Syncing the {eachvar:capitalize} repository"),
-              'method': 'Update.syncRepositories(eachvar)',
-              },
-             {'name': 'sync_reps:regen_cache',
-              'foreach': 'cl_update_sync_overlay_rep',
-              'message': __("Updating the {eachvar:capitalize} repository cache"),
-              'essential': False,
-              'method': 'Update.regenCache(eachvar)',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_metadata_force') != 'skip' or
-                               Get('cl_update_metadata_force') == 'force'))
-              },
-             {'name': 'emerge_metadata',
-              'message': __("Metadata transfer"),
-              'method': 'Update.emergeMetadata()',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_metadata_force') != 'skip' or
-                               Get('cl_update_metadata_force') == 'force'))
-              },
-             {'name': 'eix_update',
-              'message': __("Updating the eix cache for "
-                            "{cl_update_eix_repositories}"),
-              'method': 'Update.eixUpdate(cl_repository_name)',
-              'condition': (
-                  lambda Get: (Get('cl_update_outdate_set') == 'on' and
-                               Get('cl_update_eixupdate_force') != 'skip' or
-                               Get('cl_update_eixupdate_force') == 'force'))
-              },
-             # сообщение удачного завершения при обновлении репозиториев
-             {'name': 'success_syncrep',
-              'message': __("Synchronization finished"),
-              'depend': (Tasks.success() & Tasks.has_any("sync_reps",
-                                                         "sync_other_reps",
-                                                         "emerge_metadata",
-                                                         "eix_update")),
-              }
-         ]
+            {'message': __("Repository transfer"),
+             'condition': lambda Get: not (
+                 Get('cl_update_profile_storage').is_local(
+                     Get('cl_update_profile_url'),
+                     Get('cl_update_profile_branch'))) and Get('cl_update_profile_check_sync_allowed'),
+             },
+            {'name': 'reconfigure_vars1',
+             'method': 'Update.invalidateVariables("cl_update_profile_storage")',
+             },
+            {'name': 'check_datavars',
+             'error': _("Profile not found in master"),
+             'condition': lambda Get: not Get('update.cl_update_profile_datavars'),
+             },
+            {'name': 'drop_binhosts',
+             'method': 'Update.drop_binhosts(update.cl_update_profile_datavars)',
+             },
+            {'name': 'reconfigure_vars',
+             'method': 'Update.reconfigureProfileVars(cl_update_profile_datavars,'
+                              'cl_chroot_path)',
+             },
+         ],
+         'depend': Tasks.has('migrate_repository')
          },
         {'name': 'reps_synchronization',
          'group': __("Setting up the profile"),

pym/update/variables/update.py

@@ -18,6 +18,7 @@ import os
 import sys
 import re
 from os import path
+import datetime
 from calculate.lib.cl_template import SystemIni
 from calculate.lib.cl_log import log
 from calculate.lib.datavars import (Variable, VariableError,
@@ -27,7 +28,7 @@ from calculate.lib.datavars import (Variable, VariableError,
                                     SimpleDataVars, DataVarsError)
 from calculate.lib.utils.binhosts import (Binhosts, PackagesIndex, HOURS,
     BinhostSignError)
-from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin
+from calculate.lib.utils.files import readFile, listDirectory, process, pathJoin, readLinesFile
 
 from calculate.lib.configparser import ConfigParser
 from calculate.lib.cl_lang import setLocalTranslate
@@ -35,7 +36,7 @@ from calculate.lib.utils.text import simplify_profiles, _u8
 from calculate.lib.utils.git import Git, GitError
 from calculate.lib.utils.portage import ReposConf, PortageState
 from ..profile import (RepositoryStorageSet, DEFAULT_BRANCH,
-                       LocalStorage, ProfileRepository, CacheStorage)
+                       LocalStorage, ProfileRepository, CacheStorage, Profile)
 
 from calculate.lib.variables import linux as lib_linux
 from calculate.lib.variables import env
@@ -165,6 +166,60 @@ class VariableClUpdateRepUrl(Variable):
     value = []
 
 
+class VariableClUpdateRepList(Variable):
+    """
+    Список путей до репозиториев
+    """
+    type = "list"
+    value = []
+
+
+class VariableClUpdateInnerRepList(Variable):
+    value = []
+    check_after = ['cl_update_rep_list', 'cl_update_rep_hosting']
+
+    def get(self):
+        def gen():
+            val = self.Get('cl_update_rep_list') or ''
+            prev_host = self.Get('cl_update_rep_hosting') or ''
+            if prev_host:
+                for dct in val:
+                    for key, value in dct.items():
+                        if key == prev_host:
+                            yield value, key
+            for dct in val:
+                for key, value in dct.items():
+                    if key != prev_host:
+                        yield value, key
+        return [x for x in gen()]
+
+
+class VariableClUpdateRepHosting(Variable):
+    value = ''
+
+
+class VariableClUpdateRepHostingChoice(Variable):
+    type = "choice"
+    opt = ["--repo-update"]
+    metavalue = "REPOSITORY URL"
+    check_after = ['cl_update_inner_rep_list']
+
+    def init(self):
+        self.label = (_("Update repository"))
+        self.help = (_("Hosting to download repository from"))
+
+    def get(self):
+        if not self.value:
+            if self.Get('cl_update_rep_hosting'):
+                self.value = self.Get('cl_update_rep_hosting')
+            else:
+                self.value = [x[1] for x in self.Get('cl_update_inner_rep_list')][0] or ['Default']
+        return self.value
+
+    def choice(self):
+        return [x[1] for x in self.Get('cl_update_inner_rep_list')] or ['Default']
+
+
 class VariableClUpdateLaymanStorage(Variable):
     """
     Путь к репозиториям
@@ -192,17 +247,21 @@ class VariableClUpdateRepPath(ReadonlyVariable):
     Пути до репозиториев
     """
     type = "list"
-    mapPath = {'portage': 'usr/portage',
-               'gentoo': 'usr/portage'}
 
     def get(self):
         repPath = self.Get('cl_update_repos_storage')
         chroot_path = self.Get('cl_chroot_path')
+        if os.path.isdir(path.join(chroot_path, 'var/db/repos/gentoo')):
+            mapPath = {'portage': 'var/db/repos/gentoo',
+                       'gentoo': 'var/db/repos/gentoo'}
+        else:
+            mapPath = {'portage': 'usr/portage',
+                       'gentoo': 'usr/portage'}
 
         def generatePaths(names):
             for name in names:
-                if name in self.mapPath:
-                    yield path.join(chroot_path, self.mapPath[name])
+                if name in mapPath:
+                    yield path.join(chroot_path, mapPath[name])
                 else:
                     yield path.join(repPath, name)
 
@@ -284,6 +343,12 @@ class VariableClUpdateBranchRep(ReadonlyVariable):
     def get(self):
         return self.Get('cl_update_rep_name')
 
+
+class VariableClUpdateBinhostSet(Variable):
+    type = "bool"
+    value = 'off'
+
+
 class VariableClUpdateBinhostsInfo(ReadonlyVariable):
     """
     Объект для получения информации о серверах обновлений
@@ -368,6 +433,7 @@ class VariableClUpdateBinhostTime(FieldValue, Variable):
     source_variable = "cl_update_binhost_data"
     column = 3
 
+
 class VariableClUpdateBinhostLevel(FieldValue, Variable):
     """
     Список имен прочих репозиториев
@@ -392,8 +458,11 @@ class VariableClUpdateSettingsChangesSet(Variable):
     Определить были ли изменения в /etc/portage, /var/log/emerge.log
     """
     type = Variable.Types.Boolean
+    value = ''
 
     def get(self):
+        if self.value:
+            return self.value
         ini = SystemIni(self.parent)
         old_portage_state_hash = ini.getVar('system', 'portage_hash') or ""
         ps = PortageState()
@@ -401,6 +470,23 @@ class VariableClUpdateSettingsChangesSet(Variable):
         return "on" if new_portage_state_hash != old_portage_state_hash else "off"
 
 
+class VariableClUpdateSettingsPortageSet(Variable):
+    """
+    Определить были ли изменения в /etc/portage
+    """
+    type = Variable.Types.Boolean
+    value = ''
+
+    def get(self):
+        if self.value:
+            return self.value
+        ini = SystemIni(self.parent)
+        old_portage_state_hash = ini.getVar('system', 'portage_hash') or ""
+        ps = PortageState()
+        new_portage_state_hash = ps.get_state()
+        a = new_portage_state_hash != old_portage_state_hash
+        return "on" if new_portage_state_hash != old_portage_state_hash else "off"
+
 class VariableClUpdateBranchName(Variable):
     """
     Список доступных репозиторием
@@ -476,9 +562,9 @@ class VariableClUpdateOutdateSet(ReadonlyVariable):
     type = "bool"
 
     def get(self):
-        if (self.Get('cl_update_other_set') == 'on' and
-                self.Get('cl_update_other_rep_name')):
-            return "on"
+        # if (self.Get('cl_update_other_set') == 'on' and
+        #         self.Get('cl_update_other_rep_name')):
+        #     return "on"
         return "off"
 
 
@@ -580,13 +666,33 @@ class VariableClUpdateOtherSet(Variable):
     """
     Обновить остальные оверлеи
     """
-    type = "bool"
-    value = "off"
+    type = "bool3"
     opt = ["-o", "--update-other"]
+    metavalue = _("ON/OFF/AUTO")
+    value = Variable.Auto
 
     def init(self):
-        self.help = _("update other overlays")
         self.label = _("Update other overlays")
+        self.help = _("Update other overlays. Updates other overlays while updating default overlays if value is not specified")
+
+    def get(self):
+        return self.value
+
+
+class VariableClUpdateOtherGitExists(Variable):
+    """
+    Проверка, существует ли папка .git в профиле контейнера
+    """
+    type = 'bool'
+    repos_path = 'cl_update_repos_storage'
+
+    def get(self):
+        repos = self.Get(self.repos_path)
+        for rep in listDirectory(repos):
+            if not os.path.exists(os.path.join(repos, os.path.join(rep, '.git'))):
+                return False
+        return True
+
 
 
 class VariableClUpdateOtherRepData(ReadonlyTableVariable):
@@ -723,7 +829,7 @@ class VariableClUpdateRepHost(Variable):
 
 class VariableClUpdateRepHostUrl(Variable):
     type = "list"
-    value = ['git://github.com/calculatelinux/%s.git']
+    value = ['https://github.com/calculatelinux/%s.git']
 
 
 class VariableClUpdateProfileDatavars(ReadonlyVariable):
@@ -857,6 +963,16 @@ class VariableClUpdateProfileDependUrl(FieldValue, ReadonlyVariable):
         self.label = _("URL")
 
 
+class VariableClUpdateProfileCheckSyncAllowed(Variable):
+    path = 'cl_update_repos_storage'
+
+    def get(self):
+        repo_path = path.join(self.Get(self.path), self.Get('cl_update_profile_repo_name'))
+        if path.exists(repo_path) and not os.access(repo_path, os.W_OK):
+            return False
+        return True
+
+
 class VariableClUpdateProfileRepName(ReadonlyVariable):
     type = "list"
 
@@ -915,9 +1031,10 @@ class VariableClUpdateProfileUrl(Variable):
     """
     URL текущего репозитория
     """
-    untrusted = True
+    #untrusted = True
     check_after = ["cl_update_profile_branch"]
     check_action = "update_profile"
+    value = ''
 
     opt = ["--url"]
     metavalue = "URL"
@@ -925,7 +1042,6 @@ class VariableClUpdateProfileUrl(Variable):
     profile = "cl_profile_system"
     branch = "cl_update_profile_branch"
     storage = "cl_update_profile_storage"
-    default_url = "git://github.com/calculatelinux/distros.git"
     profiles_shortname = 'cl_update_profile_shortname'
 
     @property
@@ -950,7 +1066,8 @@ class VariableClUpdateProfileUrl(Variable):
     def normalize_url(cls, url):
         url = Git.normalize_url(url)
         if not url:
-            raise VariableError(_("Wrong repository URL"))
+            return ''
+            #raise VariableError(_("Wrong repository URL"))
         return url
 
     def url_by_shortname(self, value):
@@ -978,41 +1095,42 @@ class VariableClUpdateProfileUrl(Variable):
 
     def check(self, value):
         if not value:
-            raise VariableError("Need to specify profile repository")
+            return True
         try:
             branch = self.Get(self.branch)
-            self.Get(self.storage).get_profiles(value, branch)
         except GitError as e:
             raise VariableError(str(e))
         if not self.Get(self.profiles_shortname):
             raise VariableError(_("Repository %s has no profiles") %
-                value.encode('utf-8'))
-
-    def get(self):
-        try:
-            profile = self.Get(self.profile)
-            if profile:
-                while (profile != self.current_root and
-                               ".git" not in listDirectory(profile) and
-                               "repo_name" not in listDirectory(
-                               path.join(profile,"profiles"))):
-                    profile = path.dirname(profile)
-                if profile == self.current_root:
-                    return self.default_url
-                if ".git" not in listDirectory(profile):
-                    repo_name = readFile(path.join(
-                        profile,"profiles/repo_name")).strip()
-                    if (repo_name in self.rep_names and
-                        self.Get('cl_action') == self.check_action):
-                        raise CriticalError(
-                            _("You need to update the repositories before "
-                              "you change the profile"))
-                return Git.get_url(profile, "origin") or self.default_url
-        except CriticalError as e:
-            raise VariableError(str(e))
-        except Exception as e:
-            pass
-        return self.default_url
+                value)
+
+    # def get(self):
+    #     try:
+    #         if not self.value:
+    #             return self.value
+    #         profile = self.Get(self.profile)
+    #         if profile:
+    #             while (profile != self.current_root and
+    #                            ".git" not in listDirectory(profile) and
+    #                            "repo_name" not in listDirectory(
+    #                            path.join(profile,"profiles"))):
+    #                 profile = path.dirname(profile)
+    #             if profile == self.current_root:
+    #                 return self.value
+    #             if ".git" not in listDirectory(profile):
+    #                 repo_name = readFile(path.join(
+    #                     profile,"profiles/repo_name")).strip()
+    #                 if (repo_name in self.rep_names and
+    #                     self.Get('cl_action') == self.check_action):
+    #                     raise CriticalError(
+    #                         _("You need to update the repositories before "
+    #                           "you change the profile"))
+    #             return Git.get_url(profile, "origin")
+    #     except CriticalError as e:
+    #         raise VariableError(str(e))
+    #     except Exception as e:
+    #         pass
+    #     return self.value
 
 
 class VariableClUpdateProfileRepoName(ReadonlyVariable):
@@ -1044,17 +1162,48 @@ class VariableClUpdateProfileBranch(Variable):
 
     storage = "cl_update_profile_storage"
     url = "cl_update_profile_url"
+    primal_branch = "main"
+    rerun = False
 
     def init(self):
         self.label = _("Repository branch")
         self.help = _("set the repository branch")
 
     def get(self):
-        arch = self.Get("main.os_arch_machine")
-        if arch == "x86_64":
-            return "master"
+        url = self.Get(self.url)
+        if url:
+            overlay = url.split('/')[-1]
+            if overlay.endswith("-overlay.git"):
+                overlay = overlay[:-12]
+            elif overlay.endswith(".git"):
+                overlay = overlay[:-4]
         else:
-            return "master"
+            return self.primal_branch
+        ini = SystemIni(self.parent)
+        if not self.rerun:
+            branch = ini.getVar("branch", overlay)
+            if branch:
+                self.primal_branch = branch
+                self.rerun = True
+                return branch
+
+            branch = self.primal_branch
+            try:
+                self.Get(self.storage).get_profiles(url, branch)
+                self.primal_branch = branch
+            except GitError as e:
+                self.swap_primal_branch(branch)
+                branch = self.primal_branch
+                self.Get(self.storage).get_profiles(url, branch)
+
+            self.rerun = True
+            ini.setVar("branch", {overlay: self.primal_branch})
+        return self.primal_branch
+
+    def swap_primal_branch(self, branch):
+        if not self.rerun:
+            self.primal_branch = {'master': 'main', "main": 'master'}.get(branch, branch)
+        self.rerun = True
 
 
 class VariableClProfileRepository(ReadonlyVariable):
@@ -1089,19 +1238,36 @@ class VariableClProfileData(ReadonlyTableVariable):
         return [x for x in profiles if x.arch == arch]
 
     def get(self, hr=HumanReadable.No):
-        rep = self.Get(self.repository)
-        if not rep:
-            return [[]]
-        profiles = rep.get_profiles()
-        if not profiles:
+        try:
+            self.Get(f'cl_{self.section}_profile_url')
+        except:
             return [[]]
-        repo_name = profiles[0].repository.repo_name
+        if self.Get(f'cl_{self.section}_profile_url'):
+            rep = self.Get(self.repository)
+            if not rep:
+                return [[]]
+            profiles = rep.get_profiles()
+            if not profiles:
+                return [[]]
+            repo_name = profiles[0].repository.repo_name
+        else:
+            full_path = self.Get('cl_profile_system')
+            storage_path = self.Get('update.cl_update_repos_storage')
+            if self.section == 'builder':
+                full_path = pathJoin('/', self.Get('cl_builder_path'), full_path)
+            if storage_path in full_path:
+                repo_name = full_path.split(storage_path)[1].replace('/', ' ').split()[0]
+                profiles_desc = path.join(storage_path, repo_name, "profiles/profiles.desc")
+                profiles =  ProfileRepository.get_local_profiles(path.join(storage_path, repo_name))
+                for profile in profiles:
+                    profile.path = path.join(storage_path, repo_name, 'profiles', profile.profile)
+
         filtered_profiles = self.profile_filter(profiles)
         full_name = [x.profile for x in filtered_profiles]
         profile_path = [x.path for x in filtered_profiles]
         profile_arch = [x.arch for x in filtered_profiles]
         short_name = simplify_profiles(full_name)
-        full_name = ["%s:%s" % (repo_name, x) for x in full_name]
+
         return list(zip(full_name,
                    short_name,
                    profile_path,
@@ -1127,11 +1293,11 @@ class VariableClUpdateProfileRepository(ReadonlyVariable):
         try:
             rep_set = self.Get(self.storage)
             branch = self.Get(self.branch)
-            rep = rep_set.get_repository(url, branch=None)
+            rep = rep_set.get_repository(url, branch=branch)
             if rep and self.Get(self.sync_set) == 'on':
                 rep.sync()
-            return rep_set.get_repository(url, branch=None) or ""
-        except GitError:
+            return rep_set.get_repository(url, branch=branch) or ""
+        except GitError as e:
             return ""
 
 
@@ -1231,15 +1397,9 @@ class VariableClUpdateProfileSystem(Variable):
             try:
                 if (not dv.Get('cl_update_rep_name') or
                         not dv.Get('cl_update_rep_url')):
-                    raise VariableError(
-                        _("Repository variables "
-                          "were not configured for the profile"))
+                    return True
                 if not dv.Get('os_linux_name'):
                     raise VariableError("")
-                if repo_name not in list(dv.Get('cl_update_rep_name')):
-                    raise VariableError(
-                        _("Overlay %s is not specified "
-                          "in cl_update_rep_name") % repo_name)
             except (DataVarsError, VariableError) as e:
                 if str(e):
                     message = ". " + str(e)
@@ -1266,9 +1426,6 @@ class VariableClUpdateProfileSystem(Variable):
         return ""
 
     def choice(self):
-        url = self.Get(self.url)
-        if not url:
-            return []
         arch = self.Get(self.gentoo_arch)
         profiles = list(zip(*self.Select([self.profiles_shortname,
                                      self.profiles_fullname],
@@ -1322,6 +1479,8 @@ class DataVarsUpdateProfile(SimpleDataVars):
             VariableClUpdateRepName(section="update"),
             VariableClUpdateRep(section="update"),
             VariableClUpdateRepUrl(section="update"),
+            VariableClUpdateRepList(section="update"),
+            VariableClUpdateBinhostSet(section="update"),
             VariableClUpdateBinhost(section="update"),
             VariableClUpdateBinhostData(section="update"),
             VariableClUpdateBinhostHost(section="update"),
@@ -1337,6 +1496,8 @@ class DataVarsUpdateProfile(SimpleDataVars):
             VariableClUpdateBinhostStableSet(section="update"),
             VariableClUpdateBinhostStableOptSet(section="update"),
             VariableClUpdateBinhostRevisionPath(section="update"),
+            VariableClUpdateRepHosting(section="update"),
+            VariableClUpdateBinhostSet(section="update"),
         )
         self['cl_profile_system'] = profile
         self['cl_chroot_path'] = chroot_path
@@ -1575,7 +1736,7 @@ class VariableClUpdateBinhostMigrationList(Variable):
     Список хостов с бинарными обновлениями для стабильной миграции
     """
     type = "list"
-    value = ["https://mirror.calculate-linux.org/migrate/"]
+    value = ["ftp://ftp.spb.calculate.ru/pub/calculate/migration/"]
 
 class VariableClUpdateBinhostBase(Variable):
     """
@@ -1618,11 +1779,34 @@ class VariableClUpdateBinhostStableOptSet(Variable):
         return self.Get('cl_update_binhost_stable_set')
 
 
+class VariableClUpdateBinhostChoice(Variable):
+    type = "choiceedit"
+    value = ''
+    opt = ["-m", "--mirror-update"]
+    metavalue = "MIRROR_URL"
+
+    def init(self):
+        self.label = _("Packages mirror")
+        self.help = _("Url to download binary packages from")
+
+    def get(self):
+        if not self.value:
+            if self.GetBool("cl_update_binhost_set") and self.Get("cl_update_binhost"):
+                return self.Get("cl_update_binhost")
+        return self.value
+
+    def choice(self):
+        if self.GetBool("cl_update_binhost_set") and self.Get("cl_update_binhost"):
+            if self.Get("cl_update_binhost") not in self.Get("cl_update_binhost_list"):
+                return ['auto'] + [self.Get("cl_update_binhost")] + self.Get("cl_update_binhost_list")
+        return ['auto'] + self.Get("cl_update_binhost_list")
+
+
 class VariableClUpdateBinhost(Variable):
     """
     Хост с бинарными обновлениями
     """
-    value = ""
+    value = ''
 
 
 class VariableClUpdateBinhostRevisionPath(Variable):
@@ -1682,8 +1866,10 @@ class VariableClUpdateOnedepthSet(Variable):
     """
     Удлять лишние файлы из репозиториев (например созданные пользователем)
     """
-    type = "bool"
-    value = "off"
+    type = "bool3"
+    value = 'auto'
+    max_interval = 'cl_update_onedepth_interval'
+    check_after = 'cl_update_onedepth_interval'
 
     opt = ["--one-depth", "-1"]
 
@@ -1691,6 +1877,43 @@ class VariableClUpdateOnedepthSet(Variable):
         self.label = _("Clear the history of repositories")
         self.help = _("clear the history of repositories")
 
+    def get_oldest_history(self, rep):
+        try:
+            with open(f'{rep}/.git/logs/HEAD') as inf:
+                oldest_timestamp = int(inf.readline().split()[4])
+            date = datetime.datetime.fromtimestamp(oldest_timestamp)
+            return datetime.datetime.now() - date
+        except OSError:
+            return 0
+
+    def set(self, value):
+        try:
+            if value == 'auto' or value is None:
+                rep_path = self.Get('cl_update_rep_path')
+                for rep in rep_path:
+                    if self.get_oldest_history(rep).days > int(self.Get(self.max_interval)):
+                        self.value = 'on'
+                        return 'on'
+                    self.value = 'off'
+                    return 'off'
+        except:
+            return 'off'
+        self.value = value
+        return self.value
+
+    def get(self):
+        if self.value == 'auto' or self.value is None:
+            return self.set(self.value)
+        return self.value
+
+
+class VariableClUpdateOnedepthInterval(Variable):
+    default_value = 0
+
+    def get(self):
+        if not self.value:
+            return self.default_value
+        return self.value
 
 class VariableClUpdateEixRepositories(ReadonlyVariable):
     """
@@ -1752,8 +1975,11 @@ class VariableClUpdatePackageCacheSet(Variable):
     Необходимость обновить Packages
     """
     type = "bool"
+    value = ''
 
     def get(self):
+        if self.value:
+            return self.value
         packages_fn = self.Get('cl_update_package_cache')
         gpg = self.Get('cl_update_gpg')
         packages_asc_fn = self.Get('cl_update_package_cache_sign')
@@ -1867,7 +2093,7 @@ class VariableClUpdateLevel(Variable):
     Используется для выбора зеркала
     """
     type = "int"
-    value = "-1"
+    value = ""
 
 class VariableClUpdateUseMigrationHost(Variable):
     """
@@ -1876,30 +2102,65 @@ class VariableClUpdateUseMigrationHost(Variable):
     type = "bool"
     value = "off"
 
-class VariableClUpdateMigrationHost(Variable):
+
+class VariableClUpdateIgnoreLevel(Variable):
     """
-    Используемый хост миграции
+    Проверять уровень обновления 
+    при соединении с binhost
     """
-    value = ""
+    type = "bool"
+    opt = ["--ignore-level"]
+    value = "off"
+
+    def init(self):
+        self.label = _("Forcefully ignore update level and migration system")
+        self.help = _("forcefully ignore update level and migration system")
 
-class VariableClBinhostStash(Variable):
+class VariableClUpdateForceLevel(Variable):
     """
-    Бекап хоста обновления
+    Принудительное обновление до определенного уровня
     """
-    value = ""
+    type = "int"
+    opt = ["--force-level"]
+    metavalue = "LEVEL"
+
+
+    def init(self):
+        self.label = _("Force update to particular level")
+        self.help = _("force update to particular level")
 
-class VariableClCheckUpdateLevel(Variable):
+class VariableClUpdateForceLevelMode(Variable):
     """
-    Проверять уровень обновления 
-    при соединении с binhost
-    NOT IMPLEMENTED  YET
+    Режим принудительного обновления до определенного уровня
     """
     type = "bool"
-    value = "on"
+    value = "off"
 
-class VariableClUpdateMaxLevelFound(Variable):
+class VariableClUpdateSavedTag(Variable):
     """
-    Самый высокий уровень наиденный на зеркалах
+    Сохраняемый тэг репозитория calculate
     """
     type = "int"
-    value = "0"
+
+
+class VariableClUpdateSaveCurBinhost(Variable):
+    value = " "
+
+
+class VariableClUpdateWorldHashSet(Variable):
+    """
+    Хэш world для проверки изменений
+    """
+    type = Variable.Types.Boolean
+    value = ""
+
+    def get(self):
+        if self.value:
+            return self.value
+        ini = SystemIni(self.parent)
+        old_world_state_hash = ini.getVar('system', 'world_hash') or ""
+        #меняем путь сборки хэша на world файл
+        ps = PortageState()
+        ps.paths = ['/var/lib/portage/world']
+        new_world_state_hash = ps.get_state()
+        return "on" if new_world_state_hash != old_world_state_hash else "off"

pym/update/wsdl_update.py

@@ -68,28 +68,32 @@ class Wsdl(WsdlBase):
                                     normal=(
                                         'cl_update_binhost_stable_opt_set',
                                         'cl_update_binhost_recheck_set',
-                                        'cl_update_with_bdeps_opt_set'
+                                        'cl_update_with_bdeps_opt_set',
+                                        'cl_update_binhost_choice',
+                                        'cl_update_rep_hosting_choice',
                                     ),
                                     expert=(
-                                    'cl_update_sync_only_set',
-                                    'cl_update_other_set',
-                                    'cl_update_pretend_set',
-                                    'cl_update_sync_rep',
-                                    'cl_update_emergelist_set',
-                                    'cl_update_check_rep_set',
-                                    'cl_update_force_fix_set',
-                                    'cl_update_world',
-                                    'cl_update_gpg_force',
-                                    'cl_update_egencache_force',
-                                    'cl_update_eixupdate_force',
-                                    'cl_update_revdep_rebuild_set',
-                                    'cl_update_wait_another_set',
-                                    'cl_update_autocheck_schedule_set',
-                                    'cl_update_onedepth_set',
-                                    'cl_update_cleanpkg_set',
-                                    'cl_update_branch_data',
-                                    'cl_templates_locate',
-                                    'cl_verbose_set', 'cl_dispatch_conf'),
+                                        'cl_update_other_set',
+                                        'cl_update_sync_only_set',
+                                        'cl_update_pretend_set',
+                                        'cl_update_sync_rep',
+                                        'cl_update_emergelist_set',
+                                        'cl_update_check_rep_set',
+                                        'cl_update_force_fix_set',
+                                        'cl_update_ignore_level',
+                                        'cl_update_force_level',
+                                        'cl_update_world',
+                                        'cl_update_gpg_force',
+                                        'cl_update_egencache_force',
+                                        'cl_update_eixupdate_force',
+                                        'cl_update_revdep_rebuild_set',
+                                        'cl_update_wait_another_set',
+                                        'cl_update_autocheck_schedule_set',
+                                        'cl_update_onedepth_set',
+                                        'cl_update_cleanpkg_set',
+                                        'cl_update_branch_data',
+                                        'cl_templates_locate',
+                                        'cl_verbose_set', 'cl_dispatch_conf'),
                                     next_label=_("Run"))]},
         #
         # Сменить профиль
@@ -120,7 +124,7 @@ class Wsdl(WsdlBase):
             'native_error': (VariableError, DataVarsError,
                              InstallError, UpdateError, GitError),
             # значения по умолчанию для переменных этого метода
-            'setvars': {'cl_action!': 'update_profile'},
+            'setvars': {'cl_action!': 'update_profile', 'cl_update_world_default': "rebuild"},
             # описание груп (список лямбда функций)
             'groups': [
                 lambda group: group(_("Repository"),

setup.py

@@ -18,7 +18,7 @@
 #  limitations under the License.
 
 __app__ = "calculate-update"
-__version__ = "3.2.2" #TODO bump version
+__version__ = "3.7.3" #TODO bump version
 
 import os
 from glob import glob