# -*- coding: utf-8 -*- # Copyright 2010-2016 Mir Calculate. http://www.calculate-linux.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import sys import time from os import path from random import choice import string import glob import shutil from time import sleep from calculate.core.server.func import MethodsInterface from calculate.core.server.admin import Admins from calculate.lib.utils.mount import isMount from calculate.lib.utils.system import SystemPath from calculate.lib.utils.files import (pathJoin, process, listDirectory, writeFile, checkUtils, readFile, find, copyWithPath, readLinesFile, getProgPath) from collections import deque import calculate.lib.utils.device as device from calculate.lib.utils.device import (detectDeviceForPartition, countPartitions) from calculate.lib.utils.partition import DiskFactory from .datavars import DataVarsInstall from .distr import DistributiveError, PartitionDistributive from subprocess import Popen, PIPE, STDOUT from itertools import * from calculate.lib.utils.tools import traverse class InstallError(Exception): """Installation Error""" from .migrate_users import migrate from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _ setLocalTranslate('cl_install3', sys.modules[__name__]) __ = getLazyLocalTranslate(_) class Install(MethodsInterface): """Primary class for templates applying and system installation""" def __init__(self): self.clVars = None # refresh information about LVM device.lvm.refresh() # refresh information about device in udevadm info device.udev.refresh() def initVars(self, datavars=None): """Primary initialization of variables""" if not datavars: self.clVars = DataVarsInstall() self.clVars.importInstall() self.clVars.flIniFile() else: self.clVars = datavars def canInstallGrub2(self, target): """Check that system has grub2 in current and installed system""" if self.clVars.Get('os_grub2_path'): return bool([x for x in listDirectory('/var/db/pkg/sys-boot') if (x.startswith('grub-1.99') or x.startswith('grub-2'))]) return False def prepareBoot(self, target_distr): """Prepare system for boot""" if self.clVars.Get('os_install_root_type') == "flash": self.installSyslinuxBootloader(target_distr) else: if self.canInstallGrub2(target_distr): self.installGrub2Bootloader(target_distr) else: self.installLegacyGrubBootloader(target_distr) return True def setActivePartition(self, partition): """ Установка активного раздела для dos и gpt таблицы разделов """ device_name = detectDeviceForPartition(partition) if device_name is None: raise DistributiveError( _("Failed to determine the parent device for %s") % partition) # device hasn't any partition elif device_name == "": return True fdisk_cmd, gdisk_cmd, parted_cmd = checkUtils('/sbin/fdisk', '/usr/sbin/gdisk', '/usr/sbin/parted') disk = self.clVars.Select('os_install_disk_parent', where='os_install_disk_dev', eq=partition, limit=1) partition_table = self.clVars.Select('os_device_table', where='os_device_dev', eq=disk, limit=1) info = device.udev.get_device_info(name=partition) partition_number = ( info.get('ID_PART_ENTRY_NUMBER', '') or info.get('UDISKS_PARTITION_NUMBER', '')) device_partition_count = countPartitions(device_name) if device_name and not partition_number: raise DistributiveError( _("Failed to determine the partition number for %s") % partition) boot_flag = "boot" if partition_table == "dos" else "legacy_boot" if partition_table == "dos": fdisk = process(fdisk_cmd, "-l", device_name) DEVICENUM, AFLAG = 0, 1 # change_active = map( # lambda x: x[DEVICENUM], # filter( # lambda x: (x[DEVICENUM] != partition_number and # x[AFLAG] == "*" or # x[DEVICENUM] == partition_number and # not x[AFLAG] == "*"), # list(map( # lambda x: [str(x[0]), x[1][1].strip()], # # enumerate partitions # enumerate(filter(None, map( # lambda x: x.split()[:2], # # drop string before information about partitions # dropwhile( # lambda x: not x.lstrip().startswith( # "Device"), # fdisk.readlines()))))))[1:])) change_active = [x[DEVICENUM] for x in [[str(y[0]), y[1][1].strip()] for y in enumerate([z for z in [o.split()[:2] for o in dropwhile(lambda x: not x.lstrip().startswith("Device"),fdisk.readlines())] if z])][1:] if x[DEVICENUM] != partition_number and x[AFLAG] == "*" or x[DEVICENUM] == partition_number and not x[AFLAG] == "*"] else: parted = process(parted_cmd, "-m", device_name, "print") DEVICENUM, FLAGS = 0, 6 # change_active = map( # lambda x: x[DEVICENUM], filter( # lambda x: (x[DEVICENUM] != partition_number and # boot_flag in x[FLAGS].strip(';').split(', ') or # x[DEVICENUM] == partition_number and # boot_flag not in # x[FLAGS].strip(';').split(', ')), # filter(lambda x: len(x) >= 7, # map(lambda x: x.split(':'), # parted.readlines()[2:])))) change_active = [x[DEVICENUM] for x in [y for y in [z.split(':') for z in parted.readlines()[2:]] if len(y) >= 7] if x[DEVICENUM] != partition_number and boot_flag in x[FLAGS].strip(';').split(', ') or x[DEVICENUM] == partition_number and boot_flag not in x[FLAGS].strip(';').split(', ')] if not change_active: return True if partition_table == "dos": pipe = Popen([fdisk_cmd, device_name], stdin=PIPE, stdout=PIPE, stderr=PIPE) for part_num in change_active: write_str = "a\n%s\n" % part_num pipe.stdin.write(write_str.encode("UTF-8")) pipe.stdin.write(b"w\n") pipe.stdin.close() pipe.wait() elif partition_table == "gpt": pipe = Popen([gdisk_cmd, device_name], stdin=PIPE, stdout=PIPE, stderr=PIPE) if device_partition_count > 1: pipe.stdin.write(b"x\n") for part_num in change_active: write_str = "a\n%s\n2\n\n" % part_num pipe.stdin.write(write_str.encode("UTF-8")) pipe.stdin.write(b"w\nY\n") else: pipe.stdin.write(b"x\na\n2\n\nw\nY\n") pipe.stdin.close() pipe.wait() for wait_time in (0.1, 0.2, 0.5, 1, 2, 4): if path.exists(partition): return True else: sleep(wait_time) raise InstallError( _("Failed to find partition %s after changing the activity") % partition) def installSyslinuxBootloader(self, target): """ Установить syslinux загрузчик (используется для flash) """ if not self.clVars.Get('os_install_mbr'): return # прописать MBR dd_process = process("/bin/dd", "if=/usr/share/syslinux/mbr.bin", "of=%s" % self.clVars.Get('os_install_mbr')[0], stderr=STDOUT) if dd_process.failed(): raise DistributiveError( _("Failed to write the master boot record\n%s") % dd_process.read()) target.close() # выполнить установку syslinux загрузчика install_root_dev = self.clVars.Get('os_install_root_dev') syslinux_process = process("/usr/bin/syslinux", install_root_dev, stderr=STDOUT) if syslinux_process.failed(): raise DistributiveError(_("Failed to install syslinux\n%s") % syslinux_process.read()) # установить загрузочный раздел активным return self.setActivePartition(self.clVars.Get('os_install_root_dev')) def installGrub2Bootloader(self, target): """ Установка GRUB2 загрузчика """ # проверить наличие grub2 cmd_grub_install = self.clVars.Get('os_grub2_path') if not cmd_grub_install: raise DistributiveError(_("Failed to install the bootloader")) process("sync").success() # если установка GRUB2 производится на текущую систему # загруженную в builder режиме if (self.clVars.Get('os_install_scratch') == "on" and self.clVars.Get('cl_action') != "system"): prefix_boot = "/mnt/scratch" else: prefix_boot = "/" # установка UEFI if self.clVars.GetBool('os_install_uefi_set'): self.install_grub_uefi(cmd_grub_install, prefix_boot, target) # не UEFI установка else: # необходимо удалить запись /boot/efi из текущей системы # при выполнении cl-setup-boot --mbr if self.clVars.Get('os_uefi'): self.update_efi_fstab() self.mount_efi_fstab() self.install_grub_biosboot(cmd_grub_install, prefix_boot, target) def install_grub_biosboot(self, cmd_grub_install, prefix_boot, target): """ Установить GRUB, загрузчик MBR (GPT) """ # получить загрузочный раздел (если есть /boot, то # он является загрузочным иначе корень) for boot_path in ("/boot", "/"): boot_disk = self.clVars.Select("os_install_disk_dev", where="os_install_disk_mount", eq=boot_path, limit=1) if boot_disk: self.setActivePartition(boot_disk) break chroot_cmd = getProgPath('/usr/bin/chroot') chroot_dn = target.getDirectory() if chroot_dn == '/': chrooting = [] else: chrooting = [chroot_cmd, chroot_dn] # если GRUB2 версии 2.00 и выше, обычная установка требует # параметра --target=i386-pc, иначе GRUB2 может попытаться # прописать себя как UEFI if [x for x in process(*traverse([chrooting, cmd_grub_install, '--version'])) if "2." in x]: platform = ["--target=i386-pc"] else: platform = [] # прописать GRUB2 на все указанные диски targetdir = target.convertToDirectory() if not targetdir.hasSystemDirectories(): targetdir.mountSystemDirectories() try: for mbr_disk in self.clVars.Get('os_install_mbr'): grub_process = process( *traverse([chrooting, cmd_grub_install, "--boot-directory=/%s" % path.relpath(target.getBootDirectory(), chroot_dn), mbr_disk, "--force",platform]), stderr=STDOUT, envdict=os.environ) if grub_process.failed(): raise DistributiveError( _("Failed to install the bootloader")) finally: if targetdir.system_mounted: targetdir.umountSystemDirectories() def update_efi_fstab(self): """ Обновляем (или удаляем) запись /boot/efi в текущем /etc/fstab Удаление используется в случае, если выполняется обновление загрузчика в не EFI системе, хотя до этого она была установлена как EFI """ fstab_fn = pathJoin(self.clVars.Get('cl_chroot_path'), "/etc/fstab") re_efi_record = re.compile("^(# /boot/efi|\S+\s/boot/efi).*\n", flags=re.M) efidata = self.clVars.Get('os_install_fstab_efi_conf') if path.exists(fstab_fn): data = readFile(fstab_fn) m = re_efi_record.search(data) newdata = re_efi_record.sub("", data) if efidata: if m: newdata = "%s%s\n%s" % (newdata[:m.start()], efidata, newdata[m.start():]) else: newdata = "%s%s\n" % (newdata, efidata) if data != newdata: with writeFile(fstab_fn) as f: f.write(newdata) def format_efi(self): formatdisk = [(dev, fs) for dev, mp, fs, make in self.clVars.ZipVars( 'os_install_disk_dev', 'os_install_disk_mount', 'os_install_disk_format', 'os_install_disk_perform_format') if mp.startswith('/boot/efi') and make == 'on' ] if formatdisk: self.startTask(_("Formatting the partitions"), progress=True, num=len(formatdisk)) i = 1 for dev, fs in formatdisk: self.formatPartition(dev, format="vfat") self.setProgress(i) i += 1 self.endTask(True) def umountDirectory(self, directory): return PartitionDistributive(None)._umountDirectory(directory) def makeDirectory(self, directory): return PartitionDistributive(None)._makeDirectory(directory) def mountToDirectory(self, dev, mp): return PartitionDistributive(None)._mountToDirectory(dev, mp) def formatPartition(self, dev, format="ext4", label=""): return PartitionDistributive(None).formatPartition(dev, format, label) def mount_efi_fstab(self): """ Подключить /boot/efi (или отключить) согласно новым данным Используется как для подключения /boot/efi, так и для отключения. """ oldmp_efi = self.clVars.select('os_disk_mount', os_disk_mount__startswith="/boot/efi") for dev, mp in self.clVars.ZipVars('os_install_disk_dev', 'os_install_disk_mount'): if mp.startswith("/boot/efi"): curdev = isMount(mp) if curdev != dev: if curdev: self.umountDirectory(mp) self.makeDirectory(mp) self.mountToDirectory(dev, mp) if mp in oldmp_efi: oldmp_efi.remove(mp) for mp in oldmp_efi: self.umountDirectory(mp) def install_grub_uefi(self, cmd_grub_install, prefix_boot, target): if self.clVars.Get('cl_action') != "system": self.format_efi() self.update_efi_fstab() self.mount_efi_fstab() efidirs = [x for x in self.clVars.Get('os_install_disk_mount') if x.startswith('/boot/efi')] if len(efidirs) > 1: labels = ["calculate%d" % i for i in range(1, len(efidirs) + 1)] else: labels = ["calculate"] for efiname, efidir in reversed(list(zip(labels, efidirs))): self._install_grub_uefi(cmd_grub_install, prefix_boot, target, efidir, efiname) # удаляем устаревшие efi_boot_mgr = getProgPath('/usr/sbin/efibootmgr') p_efibootmgr = process(efi_boot_mgr, "-v") data = p_efibootmgr.read() for num, label in re.findall(r"Boot(\d+).*(calculate\d*)", data): if label not in labels: process(efi_boot_mgr, "-b", num, "-B").success() def _install_grub_uefi( self, cmd_grub_install, prefix_boot, target, efidir, efiname): """ Установить grub с UEFI загрузчиком """ chroot_cmd = getProgPath('/usr/bin/chroot') chroot_dn = target.getDirectory() if chroot_dn == '/': chrooting = [] else: chrooting = [chroot_cmd, chroot_dn] grub_params = [ "--boot-directory=/%s" % path.relpath(target.getBootDirectory(), chroot_dn), "--bootloader-id=%s" % efiname, "--target=x86_64-efi", "--efi-directory=%s" % efidir, "--force"] # проверяем наличие в nv-ram нужной нам записи для исключения повтора efi_boot_mgr = getProgPath('/usr/sbin/efibootmgr') efi_disk = self.clVars.Select("os_install_disk_dev", where="os_install_disk_mount", eq=efidir, limit=1) if efi_disk: efi_uuid = device.udev.get_device_info( name=efi_disk).get("ID_PART_ENTRY_UUID", "") if efi_uuid: p_efibootmgr = process(efi_boot_mgr, "-v") data = p_efibootmgr.read() if re.search(r"Boot.*{label}\s.*GPT,{uuid}.*{efipath}".format( label=efiname, uuid=efi_uuid, efipath=r"\\EFI\\%s\\grubx64.efi" % efiname), data, flags=re.M | re.I): grub_params.append("--no-nvram") # в случае установки на usb-hdd EFI загрузчик не прописывается # в efivars if self.clVars.Get('os_install_root_type') == 'usb-hdd': grub_params.append("--removable") targetdir = target.convertToDirectory() if not targetdir.hasSystemDirectories(): targetdir.mountSystemDirectories() try: grub_process = process( *traverse([ chrooting, cmd_grub_install, grub_params]), stderr=STDOUT, envdict=os.environ) if grub_process.failed(): raise DistributiveError(_("Failed to install the bootloader")) finally: if targetdir.system_mounted: targetdir.umountSystemDirectories() # проверяем успешность создания загрузочной записи # если среди загрузочных записей отсутствует запись # calculate и dmesg содержит сообщение об ошибке efivars - # запись создать не удалось dmesg = getProgPath('/bin/dmesg') if efi_boot_mgr and dmesg: if not re.search('Boot.*%s\s' % efiname, process(efi_boot_mgr).read(), re.M) and \ re.search('efivars.*set_variable.*failed', process(dmesg).read(), re.M): raise DistributiveError( _("Failed to create the UEFI boot record")) def installLegacyGrubBootloader(self, target): """ Install legecy grub boot loader Perform grub installation to disk, which has root partition """ cmd_grub = getProgPath('/sbin/grub') if not cmd_grub: raise DistributiveError(_("Failed to install the bootloader")) grub_process = process( cmd_grub, "--device-map=%s/boot/grub/device.map" % target.getDirectory(), "--batch", stderr=STDOUT) boot_disk = self.clVars.Select('os_install_disk_grub', where='os_install_disk_mount', _in=('/', '/boot'), sort='DESC', limit=1) if not boot_disk: raise DistributiveError(_("Failed to determine the boot disk")) self.setActivePartition(boot_disk) for mbr_disk in self.clVars.Get('os_install_mbr'): mbr_disk_num = self.clVars.Select("os_device_map", where="os_device_dev", eq=mbr_disk) if not mbr_disk_num and mbr_disk_num != 0: raise DistributiveError(_("Failed to determine mbr")) for line in ("root (hd%s)" % boot_disk, "setup (hd%d)" % mbr_disk_num, "quit"): grub_process.write("%s\n" % line) if grub_process.failed(): raise DistributiveError(_("Failed to install the bootloader")) def checkVideoDriver(self): """ Проверить видео драйвер, и если это nvidia, то обновить маску на пакет видеодрайвера """ if self.clVars.Get('hr_video') != 'nvidia': return True mask_file = '/etc/portage/package.mask' nvidia_mask_file = path.join(mask_file, 'nvidia') # если package.mask является файлом - делаем его директорией if path.isfile(mask_file): os.rename(mask_file, mask_file + "2") os.mkdir(mask_file, mode=0o755) os.rename(mask_file + "2", path.join(mask_file, "default")) current_nvidia_mask = readFile(nvidia_mask_file).strip() new_nvidia_mask = self.clVars.Get('os_nvidia_mask') if new_nvidia_mask == current_nvidia_mask: return True with open(nvidia_mask_file, 'w') as f: f.write(new_nvidia_mask) return True def changeScheduler(self, scheduler): """ Изменить текущий IO планировщик """ root_dev = self.clVars.Select('os_disk_parent', where='os_disk_mount', eq='/', limit=1) try: sysname = device.udev.get_syspath(name=root_dev) if device.sysfs.exists(sysname, device.sysfs.Path.BlockScheduler): device.sysfs.write( sysname, device.sysfs.Path.BlockScheduler, scheduler) except Exception: raise InstallError(_("Unable to change the I/O scheduler")) return True def clearLvm(self, devices): dv = self.clVars def get_vgs(): for pv, vg, pvbase in dv.ZipVars('os_lvm_pvname', 'os_lvm_vgname', 'os_lvm_pvname_parent'): if (pv in devices or any(x in devices for x in pvbase.split(','))): yield vg remove_vgs = set(get_vgs()) remove_pvs = set(self.clVars.select('os_lvm_pvname', os_lvm_vgname__in=remove_vgs)) remove_lvs = set(self.clVars.select('os_lvm_vgname', 'os_lvm_lvname', os_lvm_vgname__in=remove_vgs)) failed = False for vg, lv in sorted(remove_lvs): failed |= not device.lvm.remove_lv(vg, lv) for vg in sorted(remove_vgs): failed |= not device.lvm.remove_vg(vg) for pv in sorted(remove_pvs): failed |= not device.lvm.remove_pv(pv) return not failed def clearRaid(self, devices): dv = self.clVars def generate_remove_raids(): for raid, parents in dv.select('os_device_dev', 'os_device_parent', os_device_type__startswith="raid"): parents = parents.split(',') if any(x in devices for x in parents): yield raid, set(parents) remove_raids = deque(generate_remove_raids()) failed = False while remove_raids: raid, parents = remove_raids.popleft() # если среди прочих удаляемых RAID есть те, которые используют # текущий - откладываем его в конец if any(raid in _parents for _raid, _parents in remove_raids): remove_raids.append((raid, parents)) else: failed |= not device.raid.remove_raid(raid) return not failed def wait_devices(self, disks): """ Ожидание одного из указанных устройств """ for waittime in (0.1, 0.2, 0.5, 1, 2, 4): disks = [x for x in disks if x and not path.exists(x)] if not disks: break else: sleep(waittime) if disks: raise InstallError( _("Failed to found partition %s after creating " "the partition table") % ",".join(disks)) def autopartition(self, scheme_builder, devices, disks): """ Авторазметка диска входящая переменная - SchemeBuilder """ self.clearLvm(devices) self.clearRaid(devices) scheme_builder.process(DiskFactory()) self.wait_devices(disks) return True def format(self, target): """ Форматировать разделы для 'target' дистрибутива """ target.performFormat() return True def unpack(self, source, target, files_num): """ Распаковать 'source' в 'target', 'filesnum' количество копируемых файлов """ self.addProgress() if files_num.isdigit(): files_num = int(files_num) else: files_num = 0 target.installFrom(source, callbackProgress=self.setProgress, filesnum=files_num) return True def copyClt(self, source, target, cltpath): """ Скопировать clt шаблоны из 'cltpath' в 'target' дистрибутив из 'source' дистрибутива """ target_dir = target.getDirectory() source_dir = source.getDirectory() for f in (x for x in chain(*[find(pathJoin(source_dir, y), filetype="f") for y in cltpath]) if x.endswith('.clt')): copyWithPath(f, target_dir, prefix=source_dir) return True def copyOther(self, source, target): """ Скопировать прочие настройки из текущей системы в новую """ file_mask = re.compile("(/etc/ssh/ssh_host_.*|" "/root/.ssh/(id_.*|known_hosts))(?!\.old)") target_dir = target.getDirectory() source_dir = source.getDirectory() for f in (x for x in chain(*(find(pathJoin(source_dir, y), filetype="f") for y in ["/etc", "/root/.ssh"])) if file_mask.search(x)): copyWithPath(f, target_dir, prefix=source_dir) return True def rndString(self): """ Получить произвольную строку из 8 символов """ """Get random string with len 8 char""" return "".join([choice(string.ascii_letters + string.digits) for i in range(0, 8)]) def _getFreeDirectory(self, directory): """ Получить название директории """ new_dir_name = directory while path.exists(new_dir_name): new_dir_name = "%s.%s" % (directory, self.rndString()) return new_dir_name def remountNTFS(self): """ Перемонтировать NTFS разделы для работы os-prober """ res = True for disk in self.clVars.Select('os_disk_dev', where='os_disk_format', like='ntfs'): mount_dir = self._getFreeDirectory('/var/lib/calculate/mount.ntfs') try: os.mkdir(mount_dir) except (OSError, IOError): continue if process('/bin/mount', disk, mount_dir).success(): for i in (0.2, 0.5, 1, 2, 4, 5): if process('/bin/umount', mount_dir).success(): break time.sleep(i) else: self.printWARNING(_("Unable to umount %s") % mount_dir) res = False try: os.rmdir(mount_dir) except (OSError, IOError): self.printWARNING( _("Unable to remove directory %s") % mount_dir) return False return res def mountBind(self, target): """ Подключить bind точки монтирования у дистрибутива """ target.postinstallMountBind() return True def userMigrate(self, target, migrate_data, root_pwd): """ Перенос текущих пользователей в новую систему, установка пароля пользователя root """ migrator = migrate(target.getDirectory()) if not migrator.migrate([[x[0],x[2],x[3]] for x in migrate_data if x], root_pwd, [], [], ): raise InstallError(_("Failed to migrate users onto the new system")) return True def umount(self, distr): """ Отключить дистрибутив """ distr.close() return True def drop_xorg_logs(self): """ Сбросить логи загрузки xorg сервера """ for fn in glob.glob("/var/log/Xorg.*.log"): new_name = "%s.old" % fn if path.exists(new_name): os.unlink(new_name) shutil.move(fn, new_name) return True def update_admin_ini(self): """ Обновить список локальных администраторов при установке """ aliases = { 'update': 'system_update', } install_admin = Admins(self.clVars, chroot=True) install_admin.clear() for k,v in self.clVars.select('install.cl_migrate_user', 'install.cl_migrate_admin', install_cl_migrate_admin__ne=""): install_admin[k] = aliases.get(v,v) install_admin.save() return True def init_themes(self): self.clVars.Get('cl_splash_image_hash') self.clVars.Get('cl_grub_image_hash') return True