From 4c1f92d549909f16367b6ff6eba93e8489a9207e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=20=D0=94=D0=B7=D1=8E=D0=B1=D0=B5?= =?UTF-8?q?=D0=BD=D0=BA=D0=BE?= Date: Wed, 27 Oct 2021 17:54:03 +0300 Subject: [PATCH] added migration host capabilities --- pym/update/update.py | 83 ++++++++++++++++++++++++++++------ pym/update/utils/cl_update.py | 41 ++++++++++------- pym/update/variables/update.py | 21 +++++++-- 3 files changed, 111 insertions(+), 34 deletions(-) diff --git a/pym/update/update.py b/pym/update/update.py index 3aa787e..d869ce7 100644 --- a/pym/update/update.py +++ b/pym/update/update.py @@ -1489,15 +1489,17 @@ class Update(MethodsInterface): class Reason(): Success = 0 - BadSign = 1 - Outdated = 2 - Skip = 3 - Updating = 4 - WrongBinhost = 5 - BadEnv = 6 - EnvNotFound = 7 - SkipSlower = 8 - UnknownError = 9 + LevelWrong = 1 + BadSign = 2 + Outdated = 3 + Skip = 4 + Updating = 5 + WrongBinhost = 6 + BadEnv = 7 + EnvNotFound = 8 + SkipSlower = 9 + UnknownError = 10 + @staticmethod @@ -1512,7 +1514,9 @@ class Update(MethodsInterface): Update.Reason.BadSign: "FAILED (Bad sign)", Update.Reason.Skip: "SKIP", Update.Reason.SkipSlower: "", - Update.Reason.Success: "" + Update.Reason.Success: "", + #TODO formulate better + Update.Reason.LevelWrong: "FAILED (binhost level is wrong)" }.get(reason,reason) def _get_binhost_logger(self): @@ -1571,11 +1575,12 @@ class Update(MethodsInterface): reason = errors.get(status, self.Reason.UnknownError) elif binhost.bad_sign: reason = self.Reason.BadSign + elif not self.is_binhost_level_valid(binhost): + reason = self.Reason.LevelWrong else: # SUCCESS if not binhost.downgraded or stabilization: host = "-> %s" % host - #TODO add migration binhost check there reason = self.Reason.Success else: reason = self.Reason.Skip @@ -1602,10 +1607,26 @@ class Update(MethodsInterface): self.Reason.Skip, self.Reason.Updating): raise UpdateError(_("Failed to find the server with appropriate updates")) + elif actual_reason is self.Reason.LevelWrong: + self.clVars.Set("update.cl_update_use_migration_host", Variable.On) + #TODO add translation + raise UpdateError(_("Failed to find the server with appropriate level of updates")) else: raise UpdateError(_("Failed to find the working server with updates")) return retval - + + def is_binhost_level_valid(self, binhost): + return self.compare_update_level(binhost) == 0 + + # @variable_module("update") + def compare_update_level(self, binhost): + # < 0 binhost level low + # == 0 binhost level adequate + # > 0 binhost level high + if binhost.level is None: + return None + return binhost.level - int(self.clVars.Get("update.cl_update_level")) + def check_current_binhost(self, binhost_url): """ Проверка текущего сервера обновлений на валидность @@ -1614,7 +1635,8 @@ class Update(MethodsInterface): raise UpdateError(_("Current binhost is absent in list of update servers")) binhost = self.binhosts_data.get_binhost(binhost_url) - if binhost.valid and not binhost.outdated and not binhost.downgraded: + if binhost.valid and not binhost.outdated and not binhost.downgraded \ + and self.is_binhost_level_valid(binhost): if binhost.status == self.binhosts_data.BinhostStatus.Success: self.clVars.Set('update.cl_update_binhost_data', [[binhost.host, binhost.data, @@ -1626,6 +1648,13 @@ class Update(MethodsInterface): if not binhost.valid: raise UpdateError( _("Current binhost {} is not valid").format(binhost_url)) + #TODO migr + elif not self.is_binhost_level_valid(binhost): + self.clVars.Set("update.cl_update_use_migration_host", Variable.On) + #TODO translate + raise UpdateError( + _("Current binhost {} has wrong update level. Current level: {} Binhost level: {}").\ + format(binhost_url, self.clVars.Get("update.cl_update_level"), binhost.level, )) elif binhost.outdated: raise UpdateError( _("Current binhost {} is outdated").format(binhost_url)) @@ -1633,6 +1662,7 @@ class Update(MethodsInterface): raise UpdateError( _("Binary packages on the current binhost {} " "are older than local").format(binhost_url)) + if self.binhosts_data.gpg: packages_fn = self.clVars.Get('update.cl_update_package_cache') packages_sign_fn = self.clVars.Get('update.cl_update_package_cache_sign') @@ -1658,6 +1688,9 @@ class Update(MethodsInterface): binhost_url)) return True + def get_migration_mirror_url(self, host, level): + return f"{host}level{level}/" + @variable_module("update") def detect_best_binhost(self): # выполняется переход с серверов unstable обновлней на stable @@ -1670,12 +1703,34 @@ class Update(MethodsInterface): self.startTask(_("Searching new binhost")) retval = self._search_best_binhost(self.binhosts_data, stabilization) - self.clVars.Set('update.cl_update_binhost_data', retval or Variable.EmptyTable, force=True) self.endTask() return True + + def set_migration_host(self): + print("TESTTEST") + retval = [] + dv = self.clVars + last_ts = dv.Get('cl_update_last_timestamp') + binhost_list = dv.Get('cl_update_binhost_migration_list') + level = self.clVars.Get('update.cl_update_level') + binhost_list = [self.get_migration_mirror_url(x, level) for x in binhost_list] + binhosts_data = Binhosts( + # значение малозначимо, поэтому берётся из собирающей системы + dv.GetInteger('cl_update_binhost_timeout'), + dv.Get('cl_update_binhost_revision_path'), + dv.Get('cl_update_binhost_timestamp_path'), + last_ts, binhost_list, + self.get_arch_machine(), + gpg=dv.Get('update.cl_update_gpg')) + migration_host = sorted(binhosts_data.get_binhosts(), reverse=True)[0] + retval.append([migration_host.host, migration_host.data, + str(migration_host.timestamp), + str(migration_host.duration)]) + return retval + def update_rep_list(self): """ diff --git a/pym/update/utils/cl_update.py b/pym/update/utils/cl_update.py index b9ab518..bdc1f6d 100644 --- a/pym/update/utils/cl_update.py +++ b/pym/update/utils/cl_update.py @@ -78,26 +78,33 @@ def get_synchronization_tasks(object_name): 'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search") & (~AllTasks.success_one_of("check_current_binhost") | AllTasks.success_all("sync_reps"))), + 'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'off' + }, + {'name': 'set_migration_binhost', + 'method': Object('set_migration_host()'), + 'depend': (Tasks.success() & ~AllTasks.has_any("not_use_search")| + AllTasks.success_all("sync_reps")), + 'condition': lambda Get: Get('update.cl_update_use_migration_host') == 'on' }, # запасная синхронизация, в ходе которой ветки обновляются до # master - {'name': 'sync_reps_fallback', - 'foreach': 'update.cl_update_sync_rep', - 'message': - __("Fallback syncing the {eachvar:capitalize} repository"), - 'method': Object('syncRepositories(eachvar,True)'), - 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), - }, - # обновление переменных информации из binhost - {'name': 'sync_reps_fallback:update_binhost_list', - 'method': Object('update_binhost_list()'), - 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), - }, - # найти лучший сервер обновлений - {'name': 'sync_reps_fallback:detect_best_binhost', - 'method': Object('detect_best_binhost()'), - 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), - }, + # {'name': 'sync_reps_fallback', + # 'foreach': 'update.cl_update_sync_rep', + # 'message': + # __("Fallback syncing the {eachvar:capitalize} repository"), + # 'method': Object('syncRepositories(eachvar,True)'), + # 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), + # }, + # # обновление переменных информации из binhost + # {'name': 'sync_reps_fallback:update_binhost_list', + # 'method': Object('update_binhost_list()'), + # 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), + # }, + # # найти лучший сервер обновлений + # {'name': 'sync_reps_fallback:detect_best_binhost', + # 'method': Object('detect_best_binhost()'), + # 'depend': Tasks.success() & AllTasks.failed_one_of("detect_best_binhost"), + # }, {'name': 'sync_reps', 'foreach': 'update.cl_update_sync_rep', 'message': __("Checking {eachvar:capitalize} updates"), diff --git a/pym/update/variables/update.py b/pym/update/variables/update.py index c6f305f..f3a38cb 100644 --- a/pym/update/variables/update.py +++ b/pym/update/variables/update.py @@ -1561,13 +1561,13 @@ class VariableClUpdateBinhostUnstableList(Variable): value = ["https://testing.calculate-linux.org/"] # value = ["ftp://ftp.calculate-linux.ru/testing"] -class VariableClUpdateBinhostListMigration(Variable): +class VariableClUpdateBinhostMigrationList(Variable): """ Список хостов с бинарными обновлениями для стабильной миграции """ type = "list" #TODO placeholder - value = ["https://testing.calculate-linux.org/"] + value = ["https://mirror.calculate-linux.org/migrate/"] class VariableClUpdateBinhostBase(Variable): """ @@ -1858,4 +1858,19 @@ class VariableClUpdateLevel(Variable): Текущий уровень обновления Используется для выбора зеркала """ - value = "0" \ No newline at end of file + value = "1" + +class VariableClUpdateUseMigrationHost(Variable): + """ + Уровень обновления на бинхосте + """ + type = "bool" + value = "off" + +class VariableClCheckUpdateLevel(Variable): + """ + Проверять уровень обновления + при соединении с binhost + """ + type = "bool" + value = "on" \ No newline at end of file