Улучшена работа с репозиториями

pym/update/update.py

@@ -38,7 +38,7 @@ import shutil
 from collections import MutableSet
 from update_tasks import EmergeMark
 
-from calculate.lib.utils.git import Git, GitError
+from calculate.lib.utils.git import Git, GitError, MTimeKeeper
 from calculate.lib.utils.portage import (Layman, EmergeLog,
                                          EmergeLogNamedTask, PackageList,
                                          PackageInformation,
@@ -57,6 +57,7 @@ from emerge_parser import (EmergeParser, EmergeCommand, EmergeError,
 
 from calculate.lib.cl_lang import (setLocalTranslate, getLazyLocalTranslate,
                                    RegexpLocalization, _)
+
 setLocalTranslate('cl_update3', sys.modules[__name__])
 __ = getLazyLocalTranslate(_)
 
@@ -69,6 +70,7 @@ class OverlayOwnCache(MutableSet):
     """
     Сет оверлеев с интегрированным кэшем
     """
+
     def __init__(self, dv=None):
         self.dv = dv
 
@@ -118,6 +120,7 @@ def variable_module(var_env):
                 return f(self, *args, **kw)
             finally:
                 self.clVars.defaultModule = old_env
+
         return wrapper
 
     return variable_module_decor
@@ -127,6 +130,7 @@ class Update(MethodsInterface):
     """Основной объект для выполнения действий связанных с обновлением системы
 
     """
+
     def init(self):
         commandLog = path.join(self.clVars.Get('core.cl_log_path'),
                                'lastcommand.log')
@@ -146,13 +150,16 @@ class Update(MethodsInterface):
         return getProgPath(program_name)
 
     def _syncRepository(self, name, url, rpath, revision,
-                        cb_progress=None):
+                        cb_progress=None, clean=False):
         """
         Синхронизировать репозитори
         """
         dv = self.clVars
         git = Git()
         info_outdated = False
+        old_dir = "%s.old" % git._gitDir(rpath)
+        if path.exists(old_dir):
+            clean = True
         try:
             self.stash_cache(rpath, name)
             if not git.checkExistsRep(rpath):
@@ -166,15 +173,20 @@ class Update(MethodsInterface):
                     tag_cr = git.getCommit(rpath, revision)
                     cr = git.getCurrentCommit(rpath)
                     ref_type = git.reference_type(rpath, revision)
+                    if git.isNeedUnpack(rpath):
+                        need_update = True
+                    elif tag_cr != cr or ref_type == Git.Reference.Branch:
+                        need_update = True
+                    elif clean:
                         status = git.getStatusInfo(rpath)
-                    if (tag_cr != cr or ref_type == Git.Reference.Branch or
+                        if not status or status['files']:
-                        not status or status['files']):
                             need_update = True
                 except GitError:
                     need_update = True
                 if need_update:
                     git.updateTagRepository(url, rpath, revision,
-                                            cb_progress=cb_progress)
+                                            cb_progress=cb_progress,
+                                            clean=clean)
                     new_cr = git.getCurrentCommit(rpath)
                     if new_cr != cr:
                         info_outdated = True
@@ -237,7 +249,8 @@ class Update(MethodsInterface):
         Проверить повторный запуск
         """
         update_running = lambda: any(os.getpid() != x
-            for x in search_worked_process('update', dv))
+                                     for x in
+                                     search_worked_process('update', dv))
         dv = self.clVars
         if update_running():
             if not wait_update:
@@ -268,12 +281,26 @@ class Update(MethodsInterface):
                     self.endTask()
         return True
 
+    @variable_module("update")
+    def trimRepositories(self, repname):
+        """
+        Синхронизировать репозитории
+        """
+        dv = self.clVars
+        rpath = \
+            dv.select('cl_update_rep_path', cl_update_rep_name=repname, limit=1)
+        git = Git()
+        self.addProgress()
+        git.trimRepository(rpath, cb_progress=self.setProgress)
+        return True
+
     @variable_module("update")
     def syncRepositories(self, repname, clean_on_error=True):
         """
         Синхронизировать репозитории
         """
         dv = self.clVars
+        check_status = dv.GetBool('update.cl_update_check_rep_set')
         url, rpath, revision = (
             dv.Select(["cl_update_rep_url", "cl_update_rep_path",
                        "cl_update_rep_rev"],
@@ -281,6 +308,9 @@ class Update(MethodsInterface):
         if not url or not rpath:
             raise UpdateError(_("Configuration variables for repositories "
                                 "are not setup"))
+        git = Git()
+        if not git.checkUrl(url):
+            raise UpdateError(_("Git %s is unavailable") % url)
         chroot_path = path.normpath(self.clVars.Get('cl_chroot_path'))
         if chroot_path == '/':
             rpath_orig = rpath
@@ -288,6 +318,9 @@ class Update(MethodsInterface):
             rpath_orig = rpath[len(chroot_path):]
         self.addProgress()
 
+        mtime = MTimeKeeper(path.join(rpath, "profiles/updates"))
+        mtime.save()
+        try:
             if clean_on_error:
                 try:
                     layman = Layman(dv.Get('cl_update_layman_installed'),
@@ -297,7 +330,8 @@ class Update(MethodsInterface):
                     if repname != "portage":
                         layman.add(repname, url, rpath_orig)
                     if not self._syncRepository(repname, url, rpath, revision,
-                                     cb_progress=self.setProgress):
+                                                cb_progress=self.setProgress,
+                                                clean=check_status):
                         return "skip"
                     return True
                 except GitError as e:
@@ -306,12 +340,14 @@ class Update(MethodsInterface):
                     self.printWARNING(str(e))
                     self.endTask(False)
                     self.startTask(
-                    _("Re-fetching the {name} repository").format(name=repname))
+                        _("Re-fetching the {name} repository").format(
+                            name=repname))
                     self.addProgress()
                     rpath_new = "%s_new" % rpath
                     try:
                         self._syncRepository(repname, url, rpath_new, revision,
-                                         cb_progress=self.setProgress)
+                                             cb_progress=self.setProgress,
+                                             clean=check_status)
                         removeDir(rpath)
                         shutil.move(rpath_new, rpath)
                     except OSError as e:
@@ -322,7 +358,8 @@ class Update(MethodsInterface):
                         if path.exists(rpath_new):
                             removeDir(rpath_new)
             else:
-            if not self._syncRepository(repname, url, rpath, revision):
+                if not self._syncRepository(repname, url, rpath, revision,
+                                            clean=check_status):
                     return "skip"
 
             layman = Layman(dv.Get('cl_update_layman_installed'),
@@ -331,6 +368,8 @@ class Update(MethodsInterface):
                             prefix=chroot_path)
             if repname != "portage":
                 layman.add(repname, url, rpath_orig)
+        finally:
+            mtime.restore()
         return True
 
     metadata_cache_names = ("metadata/md5-cache", "metadata/cache")
@@ -410,7 +449,8 @@ class Update(MethodsInterface):
                 p = process(layman, "-s", repname, stderr=STDOUT)
             if p.failed():
                 raise UpdateError(
-                    _("Failed to update the {rname} repository").format(rname=repname),
+                    _("Failed to update the {rname} repository").format(
+                        rname=repname),
                     addon=p.read())
         finally:
             self.unstash_cache(rpath, laymanname)
@@ -660,7 +700,8 @@ class Update(MethodsInterface):
             mult = _print.bold("*")
             self.printDefault(
                 " {mult} {fullname}{flag}{shortname}{size}".format(
-                mult=mult, fullname=fullname, shortname=shortname, size=size,
+                    mult=mult, fullname=fullname, shortname=shortname,
+                    size=size,
                     flag=install_flag))
 
     def _display_install_package(self, emerge, emergelike=False):
@@ -829,8 +870,10 @@ class Update(MethodsInterface):
         emerge.installing.add_observer(self._printInstallPackage)
         emerge.uninstalling.add_observer(self._printUninstallPackage)
         emerge.fetching.add_observer(self._printFetching)
+
         def cancel_observing_fetch(fn):
             emerge.fetching.clear_observers()
+
         emerge.fetching.add_observer(cancel_observing_fetch)
         try:
             emerge.run()
@@ -962,7 +1005,6 @@ class Update(MethodsInterface):
             self.clVars.Invalidate('cl_update_profile_storage')
         return True
 
-
     def reconfigureProfileVars(self, profile_dv, chroot):
         """
         Синхронизировать репозитории
@@ -1003,7 +1045,8 @@ class Update(MethodsInterface):
                 raise UpdateError(
                     _("Failed to set the profile: %s") % _("Profile not found"))
             for rm_fn in filter(path.lexists,
-                                ('/etc/make.profile', '/etc/portage/make.profile')):
+                                ('/etc/make.profile',
+                                 '/etc/portage/make.profile')):
                 os.unlink(rm_fn)
             os.symlink(profile_path, profile_file)
         except (OSError, IOError) as e:
@@ -1016,6 +1059,7 @@ class Update(MethodsInterface):
         Наложить шаблоны из профиля
         """
         from calculate.lib.cl_template import TemplatesError, ProgressTemplate
+
         dv = DataVarsUpdate()
         try:
             dv.importUpdate()

pym/update/utils/cl_update.py

@@ -223,6 +223,13 @@ class ClUpdateAction(Action):
               'method': 'Update.syncLaymanRepository(eachvar)',
               'condition': lambda Get: Get('cl_update_other_set') == 'on'
              },
+             {'name': 'trim_reps',
+              'foreach': 'cl_update_sync_rep',
+              'message': __("Cleaning the {eachvar:capitalize} repository"),
+              'method': 'Update.trimRepositories(eachvar)',
+              'condition': lambda Get: (Get('cl_update_sync_rep') and
+                                        Get('cl_update_onedepth_set') == 'on')
+              },
              {'name': 'sync_reps:regen_cache',
               'foreach': 'cl_update_sync_overlay_rep',
               'essential': False,

pym/update/variables/update.py

@@ -1446,3 +1446,29 @@ class VariableClUpdateBinhostTimeout(Variable):
     """
     type = "int"
     value = "5"
+
+class VariableClUpdateCheckRepSet(Variable):
+    """
+    Удлять лишние файлы из репозиториев (например созданные пользователем)
+    """
+    type = "bool"
+    value = "off"
+
+    opt = ["--check-repos", "-C"]
+
+    def init(self):
+        self.label = _("Check the repositories integrity")
+        self.help = _("check and fix the repositories integrity")
+
+class VariableClUpdateOnedepthSet(Variable):
+    """
+    Удлять лишние файлы из репозиториев (например созданные пользователем)
+    """
+    type = "bool"
+    value = "off"
+
+    opt = ["--one-depth", "-1"]
+
+    def init(self):
+        self.label = _("Clear git repositories")
+        self.help = _("clear the history of git repositories")

pym/update/wsdl_update.py

@@ -78,7 +78,9 @@ class Wsdl(WsdlBase):
                                     'cl_update_wait_another_set',
                                     'cl_update_autocheck_schedule_set',
                                     'cl_update_binhost_recheck_set',
+                                    'cl_update_onedepth_set',
                                     'cl_update_cleanpkg_set',
+                                    'cl_update_check_rep_set',
                                     'cl_update_branch_data',
                                     'cl_templates_locate',
                                     'cl_verbose_set', 'cl_dispatch_conf'),