You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
calculate-utils-3-core/pym/core/backup.py

723 lines
30 KiB

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

# -*- coding: utf-8 -*-
# Copyright 2016 Mir Calculate. http://www.calculate-linux.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import stat
import re
from os import path
from calculate.core.server.core_interfaces import MethodsInterface
from calculate.lib.utils.files import (makeDirectory, removeDir, tar_directory,
FilePermission, find, listDirectory,
FilesError, process, readFileEx,
readFile, pathJoin, FindFileType)
from calculate.lib.utils.content import FileOwnersRestricted, ContentsStorage
from calculate.lib.utils.portage import getInstalledAtom, makeCfgName
from calculate.lib.configparser import ConfigParserCaseSens
from calculate.lib.cl_template import Template
from calculate.lib.utils.accounts import Passwd, Group, Shadow
from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate
from .variables.action import Actions
import tarfile
import shutil
import glob
from itertools import chain
_ = lambda x: x
setLocalTranslate('cl_core3', sys.modules[__name__])
__ = getLazyLocalTranslate(_)
class BackupError(Exception):
"""
Исключение вызванное во время резервного копирования настроек
"""
class Backup(MethodsInterface):
"""
Выполнение резервного копирования настроек
"""
def init(self):
self.apply_files = set()
self.unlink_autorun = set()
self.uid_map = {}
self.gid_map = {}
def prepare_backup(self, dn, rootname):
makeDirectory(path.join(dn, rootname))
return True
def remove_directory(self, dn):
removeDir(dn)
return True
def backup_user_changed(self, owner, dn_root):
"""
Сохранить конфигурационные файлы изменённые пользователем
:param owner:
:param dn_root:
:return:
"""
for fn in owner.get_md5_failed(lambda x: x.startswith('/etc')):
self.backup_file(fn, dn_root)
return True
def prepare_contents(self, dn, contents_file, rootname):
dn_root = path.join(dn, rootname)
fo = FileOwnersRestricted(
"/", ["/%s" % x for x in find(path.join(dn, rootname),
fullpath=False)] + ["/etc"])
self.backup_user_changed(fo, dn_root)
cs = ContentsStorage(contents_file, fo)
cs.keep(dn_root, dn_root)
return True
def create_archive(self, dn, archfile):
arch_dn = path.dirname(archfile)
if not path.exists(arch_dn):
makeDirectory(arch_dn)
os.chmod(arch_dn, FilePermission.UserAll)
tar_directory(dn, archfile)
return True
def open_archive(self, dn, archfile):
makeDirectory(dn)
with tarfile.open(archfile, 'r:bz2') as f:
f.extractall(dn)
return True
def restore_configs(self, archfile, dn, contents_name, root_name):
"""
Восстановить все файлы настроек
:param archfile:
:param dn:
:return:
"""
dirs_data = {}
used_dirs = set()
_gid = lambda x: self.gid_map.get(x, x)
_uid = lambda x: self.uid_map.get(x, x)
with tarfile.open(archfile, 'r:bz2') as f:
try:
# исключить из переноса файлы, которые принадлежат пакетам,
# которые не установлены в системе
contents = f.extractfile(f.getmember(contents_name))
pkg_file = [x.split()[:3:2] for x in contents]
not_installed_files = [
x for x in pkg_file
if not any(getInstalledAtom(x[0].partition(":")[0]))]
skip_packages = sorted(list(
set([x[0] for x in not_installed_files])))
if skip_packages:
self.printWARNING(
_("Settings ignored for following packages: %s") %
", ".join(x.partition(":")[0] for x in skip_packages))
not_installed_files = [x[1] for x in not_installed_files]
except KeyError:
raise BackupError(_("CONTENTS file not found"))
for ti in (x for x in f if x.name.startswith("%s/" % root_name)):
if ti.name[4:] in not_installed_files:
continue
if ti.issym() and not path.exists(ti.linkpath):
continue
if ti.name[5:]:
fn_system = path.join(dn, ti.path[5:])
if ti.isdir():
dirs_data[fn_system] = (ti.mode, _uid(ti.uid),
_gid(ti.gid))
continue
dirs_list = fn_system.split('/')
for i in range(2, len(dirs_list)):
used_dirs.add("/".join(dirs_list[:i]))
if path.lexists(fn_system):
stat_system = os.lstat(fn_system)
if ti.issym():
if stat.S_ISLNK(stat_system.st_mode):
system_link = os.readlink(fn_system)
if system_link == ti.linkpath:
continue
else:
if (stat.S_IMODE(stat_system.st_mode) == ti.mode and
stat_system.st_uid == _uid(ti.uid) and
stat_system.st_gid == _gid(ti.gid)):
data_system = readFile(fn_system)
extr_file = f.extractfile(ti)
if extr_file:
data_ti = extr_file.read()
if self.is_equal_files(data_system,
data_ti):
continue
ti.name = ti.name[5:]
f.extract(ti, dn)
os.chown(fn_system, _uid(ti.uid), _gid(ti.gid))
if ti.isfile() or ti.issym():
# если симлинк в списке предварительного удаления
# то исключаем его из списка изменённых файлов
if fn_system in self.unlink_autorun:
self.unlink_autorun.remove(fn_system)
else:
self.apply_files.add(fn_system)
# восстановление прав у каталогов, конфиги в которых должны были
# восстанавливаться
for dn_name in sorted(used_dirs):
if dn_name in dirs_data:
dn_mode, dn_uid, dn_gid = dirs_data[dn_name]
if path.lexists(dn_name):
stat_system = os.lstat(dn_name)
if (stat.S_IMODE(stat_system.st_mode) != dn_mode or
stat_system.st_uid != dn_uid or
stat_system.st_gid != dn_gid):
os.chmod(dn_name, dn_mode)
os.chown(dn_name, dn_uid, dn_gid)
return True
def sava_ini(self, section, key, val):
ini = ConfigParserCaseSens(strict=False)
ini.read(self.clVars.Get('cl_backup_ini_env'), encoding="utf-8")
if not ini.has_section(section):
ini.add_section(section)
ini[section][key] = str(val)
ini["backup"]["version"] = self.clVars.Get('cl_ver')
with open(self.clVars.Get('cl_backup_ini_env'), 'w') as f:
ini.write(f)
return True
def load_ini(self, section, key):
ini = ConfigParserCaseSens(strict=False)
ini.read(self.clVars.Get('cl_backup_ini_env'), encoding="utf-8")
return ini.get(section, key, fallback="")
def save_initd(self, dn, root_name):
"""
Сохранить список init.d
:param dn:
:param root_name:
:return:
"""
self.sava_ini("backup", "init",
','.join(listDirectory('/etc/init.d', fullPath=False)))
dn_root = path.join(dn, root_name)
for dn in ('/etc/runlevels/sysinit',
'/etc/runlevels/default',
'/etc/runlevels/boot'):
try:
dn_backup = pathJoin(dn_root, dn)
if not path.exists(dn_backup):
makeDirectory(dn_backup)
for fn in listDirectory(dn, fullPath=True):
if path.islink(fn):
link = os.readlink(fn)
symname = pathJoin(dn_root, fn)
if not path.lexists(symname):
os.symlink(link, symname)
except (OSError, IOError) as e:
raise BackupError(_("Failed to enable service at startup") +
(_(": %s") % (str(e))))
return True
def make_directory_sync(self, base_dn, dn, prefix="/"):
"""
Создать директорию и сохранить права из prefix
:param dn:
:param prefix:
:return:
"""
if not path.exists(dn):
self.make_directory_sync(base_dn, path.dirname(dn), prefix)
rel_dn = path.relpath(dn, base_dn)
system_dn = path.join(prefix, rel_dn)
system_dn_stat = os.lstat(system_dn)
if not makeDirectory(dn):
raise BackupError(_("Failed to create directory %s") % dn)
os.chown(dn, system_dn_stat.st_uid, system_dn_stat.st_gid)
os.chmod(dn, stat.S_IMODE(system_dn_stat.st_mode))
def backup_file(self, source_fn, target_dn, prefix="/"):
"""
Сделать резервную копию указанного файла
:param source_fn:
:param target_dn:
:return:
"""
target_fn = path.join(target_dn, path.relpath(source_fn, prefix))
source_stat = os.lstat(source_fn)
target_base_dn = path.dirname(target_fn)
self.make_directory_sync(target_dn, target_base_dn, prefix=prefix)
if stat.S_ISLNK(source_stat.st_mode):
source_link = os.readlink(source_fn)
os.symlink(source_link, target_fn)
elif stat.S_ISREG(source_stat.st_mode):
shutil.copy2(source_fn, target_fn)
os.chown(target_fn, source_stat.st_uid, source_stat.st_gid)
return True
def backup_marked(self, source_dn, target_dn, subdn, root_name):
"""
Сохранить файлы из указанного каталога, отмеченного комментариями
выполнения шаблонов
:return:
"""
source_etc_dn = path.join(source_dn, subdn)
root_dn = path.join(target_dn, root_name)
reCfg = re.compile('._cfg\d{4}_')
try:
for fn in find(source_etc_dn, filetype=FindFileType.RegularFile):
if (not reCfg.search(fn) and
b" Modified Calculate" in readFileEx(fn,
headbyte=300)):
self.backup_file(fn, root_dn, prefix=source_dn)
except (OSError, IOError) as e:
raise BackupError(_("Failed to backup configuration files that "
"were modified by templates") +
(_(": %s") % (str(e))))
return True
def clear_autorun(self):
"""
Удалить все файлы из автозапуска, которые ссылаются на файлы из списка
init.d
:return:
"""
files = ["/etc/init.d/%s" % x.strip()
for x in self.load_ini("backup", "init").split(',')]
for dn in ('/etc/runlevels/sysinit',
'/etc/runlevels/default',
'/etc/runlevels/boot'):
for fn in listDirectory(dn, fullPath=True):
if path.islink(fn) and os.readlink(fn) in files:
os.unlink(fn)
self.unlink_autorun.add(fn)
return True
def restore_contents(self, contentsfile, dn):
cs = ContentsStorage(contentsfile)
cs.restore(dn, files=self.apply_files)
return True
def set_service_action(self):
self.clVars.Set('core.cl_backup_action', Actions.Service, force=True)
return True
nm_name = "NetworkManager"
nm_config = "/etc/NetworkManager"
nm_connections = path.join(nm_config, "system-connections")
def _do_service(self, service, action):
"""
Выполнить действие с сервисом (restart, start, stop, zap)
:param service:
:param action:
:return:
"""
actions = {
'restart': _("Failed to restart {name} service"),
'start': _("Failed to start {name} service"),
'stop': _("Failed to stop {name} service"),
'zap': _("Failed to zap {name} service"),
'status': _("Failed to get status of {name} service")
}
try:
p = process(service, action)
if p.failed():
data = p.readerr().strip()
if ("has started, but is inactive" not in data and
"will start when" not in data):
for line in data.split('\n'):
self.printERROR(line)
raise BackupError(actions.get(action, action).format(
name=path.basename(service)))
except FilesError:
self.printERROR(actions.get(action, action).format(
name=path.basename(service)))
raise
return True
def stop_net_services(self):
"""
Остановить все сетевые службы (и NM и openrc)
:return:
"""
self._do_service("/etc/init.d/netmount", "zap")
for fn in chain(["/etc/init.d/%s" % self.nm_name],
glob.glob('/etc/init.d/net.*')):
if fn.endswith('.lo') or not path.exists(fn):
continue
self._do_service(fn, "stop")
return True
def unlink_openrc_net_services(self, files):
"""
Удалить сетевые сервисы openrc сервисы openrc
:return:
"""
for fn in glob.glob('/etc/init.d/net.*'):
if fn.endswith('.lo') or not path.exists(fn) or fn in files:
continue
try:
os.unlink(fn)
self.apply_files.add(fn)
except OSError as e:
self.printERROR(_("Failed to remove %s service"),
path.basename(fn))
self.printERROR(str(e))
return True
def is_networkmanager_backup(self, backup_path):
"""
Проверить сетевой менеджер в резервной копии
:param backup_path:
:return:
"""
return path.lexists(path.join(
backup_path, "root/etc/runlevels/default/%s" % self.nm_name))
def is_networkmanager_system(self):
"""
Проверить сетевой менеджер в текущей системе
:return:
"""
return path.lexists("/etc/runlevels/default/%s" % self.nm_name)
def restore_openrc_net_initd(self, files):
"""
Восстановить сервисы net.* и запустить их
:return:
"""
for fn in files:
if not path.exists(fn):
os.symlink("/etc/init.d/net.lo", fn)
self.apply_files.add(fn)
self._do_service(fn, "start")
return True
def restore_files(self, backup_path, files, notapply=False):
"""
Восстановить указанные файлы из backup/root
:param backup_path:
:param files: список файлов (поддерживаются глобальные символы)
:return:
"""
len_source_prefix = len(path.join(backup_path, "root"))
_gid = lambda x: self.gid_map.get(x, x)
_uid = lambda x: self.uid_map.get(x, x)
for source in chain(*[glob.glob(pathJoin(backup_path, "root", x))
for x in files]):
dest = source[len_source_prefix:]
if path.lexists(source):
dn = path.dirname(dest)
if not path.exists(dn):
makeDirectory(dn)
if path.lexists(dest):
if self.is_equal_system_backup(dest, source):
continue
shutil.copy2(source, dest)
fn_stat = os.lstat(source)
os.chown(dest, _uid(fn_stat.st_uid), _gid(fn_stat.st_gid))
if not notapply:
self.apply_files.add(dest)
return True
def clear_nm_connections(self, backup_path):
"""
Удалить доступные соединения для NetworkManager
:return:
"""
base_dir = pathJoin(backup_path, "root")
for fn in listDirectory(self.nm_connections, fullPath=True):
try:
if not path.exists(pathJoin(base_dir, fn)):
os.unlink(fn)
self.apply_files.add(fn)
except OSError as e:
raise BackupError(str(e))
def is_equal_files(self, text1, text2):
"""
Сравнить два файла отбросив комментарии и пробельные символы в начале и
в конце
:param text1:
:param text2:
:return:
"""
text1 = Template.removeComment(text1).strip()
text2 = Template.removeComment(text2).strip()
return text1 == text2
def is_equal_system_backup(self, system_fn, backup_fn):
"""
Проверить одинаковый ли файл в резервной копии и системе
:param system_fn:
:param backup_fn:
:return:
"""
_gid = lambda x: self.gid_map.get(x, x)
_uid = lambda x: self.uid_map.get(x, x)
if path.islink(system_fn) != path.islink(backup_fn):
return False
if path.islink(system_fn):
return os.readlink(system_fn) == os.readlink(backup_fn)
if path.isfile(system_fn) != path.isfile(backup_fn):
return False
data_system = readFile(system_fn)
data_backup = readFile(backup_fn)
if not self.is_equal_files(data_system, data_backup):
return False
stat_system = os.lstat(system_fn)
stat_backup = os.lstat(backup_fn)
if (stat.S_IMODE(stat_system.st_mode) !=
stat.S_IMODE(stat_backup.st_mode) or
stat_system.st_uid != _uid(stat_backup.st_uid) or
stat_system.st_gid != _gid(stat_backup.st_gid)):
return False
return True
def check_backup_for_network(self, backup_path, files):
"""
Проверить конфигурационные файлы настройки сети на соответствие текущим
:param backup_path:
:return:
"""
backup_nm = self.is_networkmanager_backup(backup_path)
system_nm = self.is_networkmanager_system()
# проверить совпадает ли сетевой менеджер
if backup_nm != system_nm:
if backup_nm and not any(
getInstalledAtom("net-misc/networkmanager")):
return True
return False
# если nm проверить совпадение system-connections
if backup_nm:
connection_files = set(path.basename(x) for x in chain(
glob.glob("%s/*" % self.nm_connections),
glob.glob("%s/*" % (pathJoin(backup_path, "root",
self.nm_connections)))
))
for fn in connection_files:
system_fn = pathJoin(self.nm_connections, fn)
backup_fn = pathJoin(backup_path, "root",
self.nm_connections, fn)
if not self.is_equal_system_backup(system_fn, backup_fn):
return False
# если openrc проверить conf.d/net и соответствие net.*
else:
system_fn = "/etc/conf.d/net"
backup_fn = pathJoin(backup_path, "root", system_fn)
if not self.is_equal_system_backup(system_fn, backup_fn):
return False
system_net_set = (set(path.basename(x)
for x in glob.glob('/etc/init.d/net.*')) -
{"net.lo"})
backup_net_set = set(path.basename(x) for x in files)
if system_net_set != backup_net_set:
return False
return True
def restore_network(self, backup_path):
"""
Восстановить сеть из backup
:param backup_path:
:return:
"""
files = ["/etc/init.d/%s" % x.strip()
for x in self.load_ini("backup", "init").split(',')
if x.startswith("net.") and x != "net.lo"]
if self.check_backup_for_network(backup_path, files):
self.endTask("skip")
return True
self.stop_net_services()
self.unlink_openrc_net_services(files)
self.clear_nm_connections(backup_path)
self.restore_files(backup_path, ["/etc/conf.d/hostname",
"/etc/resolv.conf",
"/etc/hosts"])
self._do_service("/etc/init.d/hostname", "restart")
if self.is_networkmanager_backup(backup_path):
self.unlink_openrc_net_services([])
self.restore_files(backup_path, [
"/etc/NetworkManager/system-connections/*",
"/etc/NetworkManager/dispatcher.d/*",
])
self._do_service("/etc/init.d/NetworkManager", "start")
else:
self.restore_files(backup_path, ["/etc/conf.d/net"])
self.restore_openrc_net_initd(files)
self._do_service("/etc/init.d/netmount", "start")
return True
def special_backup(self, backup_path):
"""
Выполнить специализирование резервное копирование модулей
:param backup_path:
:return:
"""
for backup_obj in self.iterate_modules():
backup_obj.backup(backup_path)
return True
def special_restore(self, backup_path):
"""
Выполнить специализирование восстановление из резервной копии
:param backup_path:
:return:
"""
for backup_obj in self.iterate_modules():
backup_obj.restore(backup_path)
return True
def iterate_modules(self):
"""
Перебрать все модули backup
:return:
"""
site_packages = [path.join(x, "calculate")
for x in sys.path
if (x.endswith('site-packages') and
x.startswith('/usr/lib'))]
ret_list = []
for module, modDir in chain(*(((path.basename(y), y) for y
in listDirectory(x, True, True)) for x in site_packages)):
if path.exists(path.join(modDir, "backup_%s.py" % module)):
if not "calculate-%s" % module in ret_list:
ret_list.append("calculate-%s" % module)
cl_backup = ret_list
for pack in cl_backup:
if pack:
module_name = '%s.backup_%s' % (pack.replace("-", "."),
pack.rpartition("-")[2])
import importlib
try:
backup_module = importlib.import_module(module_name)
backup_obj = backup_module.Backup(self, self.clVars)
yield backup_obj
except ImportError:
sys.stderr.write(_("Unable to import %s") % module_name)
def display_changed_configs(self):
"""
Отобразить список восстановленных файлов
:return:
"""
t = Template(self.clVars, printWARNING=self.printWARNING,
printERROR=self.printERROR, printSUCCESS=self.printSUCCESS)
t.verboseOutput(sorted(list(self.apply_files | self.unlink_autorun)))
return True
def display_backup_configs(self, archfile):
"""
Отобразить список помещённых в резервную копию файлов
:return:
"""
with tarfile.open(archfile, 'r:bz2') as f:
self.printWARNING(_("Calculate Utilities have backuped files")
+ _(":"))
for fn in sorted("/%s" % x.path.partition('/')[2] for x in
f.getmembers() if (not x.isdir() and (
x.path.startswith("root") or
x.path.startswith("ldap")))):
self.printSUCCESS(" " * 5 + fn)
return True
def run_openrc(self, command):
p = process("/sbin/openrc", "default")
p.success()
return True
passwd_fn = '/etc/passwd'
group_fn = '/etc/group'
shadow_fn = '/etc/shadow'
def save_accounts(self, backup_path):
accounts_path = path.join(backup_path, "accounts")
for source_fn in (self.passwd_fn, self.group_fn, self.shadow_fn):
self.backup_file(source_fn, accounts_path, prefix="/etc")
return True
def restore_accounts(self, backup_path):
accounts_path = path.join(backup_path, "accounts")
backup_passwd_fn = pathJoin(accounts_path,
path.basename(self.passwd_fn))
backup_group_fn = pathJoin(accounts_path, path.basename(self.group_fn))
backup_shadow_fn = pathJoin(accounts_path,
path.basename(self.shadow_fn))
if any(not path.exists(x) for x in (backup_passwd_fn,
backup_group_fn,
backup_shadow_fn)):
return "skip"
# пользователи
passwd = Passwd(readFile(self.passwd_fn))
backup_passwd = Passwd(readFile(backup_passwd_fn))
added_users = [x.name for x in passwd.new_users(backup_passwd)]
keep_users = [x.name for x in backup_passwd.new_users(passwd)]
if self.clVars.GetBool('cl_backup_verbose_set') and added_users:
self.printSUCCESS(
_("Restored users:") + " " + ", ".join(added_users))
self.uid_map = backup_passwd.get_uid_map(passwd)
passwd.join(backup_passwd)
with open(makeCfgName(self.passwd_fn), 'w') as f:
passwd.write(f)
os.chown(self.passwd_fn, 0, 0)
os.chmod(self.passwd_fn,
FilePermission.OtherRead |
FilePermission.GroupRead |
FilePermission.UserRead |
FilePermission.UserWrite)
# группы
groups = Group(readFile(self.group_fn))
backup_groups = Group(readFile(backup_group_fn))
added_groups = [x.name for x in groups.new_groups(backup_groups)]
if self.clVars.GetBool('cl_backup_verbose_set') and added_groups:
self.printSUCCESS(_("Restored groups:") + " "
+ ", ".join(added_groups))
self.gid_map = backup_groups.get_gid_map(groups)
groups.join(backup_groups, keep_users=keep_users)
with open(makeCfgName(self.group_fn), 'w') as f:
groups.write(f)
os.chown(self.group_fn, 0, 0)
os.chmod(self.group_fn,
FilePermission.OtherRead |
FilePermission.GroupRead |
FilePermission.UserRead |
FilePermission.UserWrite)
# пароли
shadow = Shadow(readFile(self.shadow_fn))
backup_shadow = Shadow(readFile(backup_shadow_fn))
changed_shadow = [x.name
for x in shadow.changed_passwords(backup_shadow)]
if self.clVars.GetBool('cl_backup_verbose_set') and changed_shadow:
self.printSUCCESS(_("Restored user passwords:") + " "
+ ", ".join(changed_shadow))
shadow.join(backup_shadow)
with open(makeCfgName(self.shadow_fn), 'w') as f:
shadow.write(f)
os.chown(self.shadow_fn, 0, 0)
os.chmod(self.shadow_fn,
FilePermission.UserRead |
FilePermission.UserWrite)
return True