calculate-utils-3-install/pym/install/install.py

# -*- coding: utf-8 -*-

# Copyright 2010-2016 Mir Calculate. http://www.calculate-linux.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re
import sys
import time
from os import path
from random import choice
import string
import glob
import shutil
from time import sleep
from calculate.core.server.func import MethodsInterface
from calculate.lib.utils.files import (pathJoin,
                                       isMount, process, listDirectory,
                                       checkUtils, readFile, find, copyWithPath,
                                       readLinesFile, getProgPath)
from calculate.lib.utils.device import (detectDeviceForPartition,
                                        getUdevDeviceInfo, refreshLVM,
                                        refreshUdev, countPartitions)
from datavars import DataVarsInstall
from calculate.install.variables.autopartition import AutoPartition

from distr import DistributiveError
from subprocess import Popen, PIPE, STDOUT
from itertools import *


class InstallError(Exception):
    """Installation Error"""


from migrate_users import migrate

from calculate.lib.cl_lang import setLocalTranslate, getLazyLocalTranslate, _

setLocalTranslate('cl_install3', sys.modules[__name__])
__ = getLazyLocalTranslate(_)


class Install(MethodsInterface):
    """Primary class for templates applying and system installation"""

    def __init__(self):
        self.clVars = None
        # refresh information about LVM
        refreshLVM()
        # refresh information about device in udevadm info
        refreshUdev()

    def initVars(self, datavars=None):
        """Primary initialization of variables"""
        if not datavars:
            self.clVars = DataVarsInstall()
            self.clVars.importInstall()
            self.clVars.flIniFile()
        else:
            self.clVars = datavars

    def canInstallGrub2(self, target):
        """Check that system has grub2 in current and installed system"""
        if self.clVars.Get('os_grub2_path'):
            return bool(
                filter(lambda x: (x.startswith('grub-1.99') or
                                  x.startswith('grub-2')),
                       listDirectory('/var/db/pkg/sys-boot')))
        return False

    def prepareBoot(self, target_distr):
        """Prepare system for boot"""
        if self.clVars.Get('os_install_root_type') == "flash":
            self.installSyslinuxBootloader(target_distr)
        else:
            if self.canInstallGrub2(target_distr):
                self.installGrub2Bootloader(target_distr)
            else:
                self.installLegacyGrubBootloader(target_distr)
        return True

    def setActivePartition(self, partition):
        """
        Установка активного раздела для dos и gpt таблицы разделов
        """
        device_name = detectDeviceForPartition(partition)
        if device_name is None:
            raise DistributiveError(
                _("Failed to determine the parent device for %s") % partition)
        # device hasn't any partition
        elif device_name == "":
            return True
        fdisk_cmd, gdisk_cmd, parted_cmd = checkUtils('/sbin/fdisk',
                                                      '/usr/sbin/gdisk',
                                                      '/usr/sbin/parted')

        disk = self.clVars.Select('os_install_disk_parent',
                                  where='os_install_disk_dev', eq=partition,
                                  limit=1)
        partition_table = self.clVars.Select('os_device_table',
                                             where='os_device_dev',
                                             eq=disk, limit=1)
        partition_number = (
            getUdevDeviceInfo(name=partition).get('ID_PART_ENTRY_NUMBER', '') or
            getUdevDeviceInfo(name=partition).get('UDISKS_PARTITION_NUMBER', '')
        )

        device_partition_count = countPartitions(device_name)
        if device_name and not partition_number:
            raise DistributiveError(
                _("Failed to determine the partition number for %s")
                % partition)
        boot_flag = "boot" if partition_table == "dos" else "legacy_boot"
        if partition_table == "dos":
            fdisk = process(fdisk_cmd, "-l", device_name)
            DEVICENUM, AFLAG = 0, 1
            change_active = map(
                lambda x: x[DEVICENUM],
                filter(
                    lambda x: (x[DEVICENUM] != partition_number and
                               x[AFLAG] == "*" or
                               x[DEVICENUM] == partition_number and
                               not x[AFLAG] == "*"),
                    list(map(
                        lambda x: [str(x[0]), x[1][1].strip()],
                        # enumerate partitions
                        enumerate(filter(None, map(
                            lambda x: x.split()[:2],
                            # drop string before information about partitions
                            dropwhile(
                                lambda x: not x.lstrip().startswith(
                                    "Device"),
                                fdisk.readlines()))))))[1:]))
        else:
            parted = process(parted_cmd, "-m", device_name, "print")
            DEVICENUM, FLAGS = 0, 6
            change_active = map(
                lambda x: x[DEVICENUM], filter(
                    lambda x: (x[DEVICENUM] != partition_number and
                               boot_flag in x[FLAGS].strip(';').split(', ') or
                               x[DEVICENUM] == partition_number and
                               boot_flag not in
                               x[FLAGS].strip(';').split(', ')),
                    filter(lambda x: len(x) >= 7,
                           map(lambda x: x.split(':'),
                               parted.readlines()[2:]))))
        if not change_active:
            return True
        if partition_table == "dos":
            pipe = Popen([fdisk_cmd, device_name],
                         stdin=PIPE, stdout=PIPE, stderr=PIPE)
            for part_num in change_active:
                pipe.stdin.write("a\n%s\n" % part_num)
            pipe.stdin.write("w\n")
            pipe.stdin.close()
            pipe.wait()
        elif partition_table == "gpt":
            pipe = Popen([gdisk_cmd, device_name],
                         stdin=PIPE, stdout=PIPE, stderr=PIPE)
            if device_partition_count > 1:
                pipe.stdin.write("x\n")
                for part_num in change_active:
                    pipe.stdin.write("a\n%s\n2\n\n" % part_num)
                pipe.stdin.write("w\nY\n")
            else:
                pipe.stdin.write("x\na\n2\n\nw\nY\n")
            pipe.stdin.close()
            pipe.wait()
        for wait_time in (0.1, 0.2, 0.5, 1, 2, 4):
            if path.exists(partition):
                return True
            else:
                sleep(wait_time)
        raise InstallError(
            _("Failed to find partition %s after changing the activity") %
            partition)

    def installSyslinuxBootloader(self, target):
        """
        Установить syslinux загрузчик (используется для flash)
        """
        if not self.clVars.Get('os_install_mbr'):
            return
        # прописать MBR
        dd_process = process("/bin/dd", "if=/usr/share/syslinux/mbr.bin",
                             "of=%s" % self.clVars.Get('os_install_mbr')[0],
                             stderr=STDOUT)
        if dd_process.failed():
            raise DistributiveError(
                _("Failed to write the master boot record\n%s") %
                dd_process.read())
        target.close()
        # выполнить установку syslinux загрузчика
        install_root_dev = self.clVars.Get('os_install_root_dev')
        syslinux_process = process("/usr/bin/syslinux",
                                   install_root_dev, stderr=STDOUT)
        if syslinux_process.failed():
            raise DistributiveError(_("Failed to install syslinux\n%s") %
                                    syslinux_process.read())
        # установить загрузочный раздел активным
        return self.setActivePartition(self.clVars.Get('os_install_root_dev'))

    def installGrub2Bootloader(self, target):
        """
        Установка GRUB2 загрузчика
        """
        # проверить наличие grub2
        cmd_grub_install = self.clVars.Get('os_grub2_path')
        if not cmd_grub_install:
            raise DistributiveError(_("Failed to install the bootloader"))
        process("sync").success()
        # если установка GRUB2 производится на текущую систему
        # загруженную в builder режиме
        if (self.clVars.Get('os_install_scratch') == "on" and
                    self.clVars.Get('cl_action') != "system"):
            prefix_boot = "/mnt/scratch"
        else:
            prefix_boot = "/"
        # установка UEFI
        if self.clVars.GetBool('os_install_uefi_set'):
            self.install_grub_uefi(cmd_grub_install, prefix_boot, target)
        # не UEFI установка
        else:
            self.install_grub_biosboot(cmd_grub_install, prefix_boot, target)

    def install_grub_biosboot(self, cmd_grub_install, prefix_boot, target):
        """
        Установить GRUB, загрузчик MBR (GPT)
        """
        # получить загрузочный раздел (если есть /boot, то
        # он является загрузочным иначе корень)
        for boot_path in ("/boot", "/"):
            boot_disk = self.clVars.Select("os_install_disk_dev",
                                           where="os_install_disk_mount",
                                           eq=boot_path, limit=1)
            if boot_disk:
                self.setActivePartition(boot_disk)
                break

        # если GRUB2 версии 2.00 и выше, обычная установка требует
        # параметра --target=i386-pc, иначе GRUB2 может попытаться
        # прописать себя как UEFI
        if filter(lambda x: "2." in x,
                  process(cmd_grub_install, '--version')):
            platform = ["--target=i386-pc"]
        else:
            platform = []
        # прописать GRUB2 на все указанные диски
        for mbr_disk in self.clVars.Get('os_install_mbr'):
            grub_process = process(cmd_grub_install,
                                   "--boot-directory=%s" % pathJoin(
                                       prefix_boot,
                                       target.getBootDirectory()),
                                   mbr_disk, "--force", *platform,
                                   stderr=STDOUT, envdict=os.environ)
            if grub_process.failed():
                raise DistributiveError(
                    _("Failed to install the bootloader"))

    def install_grub_uefi(self, cmd_grub_install, prefix_boot, target):
        """
        Установить grub с UEFI загрузчиком
        """
        grub_params = [
            "--boot-directory=%s" % pathJoin(
                prefix_boot,
                target.getBootDirectory()),
            "--target=x86_64-efi",
            "--efi-directory=%s" %
            target.getEfiDirectory(),
            "--force"]
        # в случае установки на usb-hdd EFI загрузчик не прописывается
        # в efivars
        if self.clVars.Get('os_install_root_type') == 'usb-hdd':
            grub_params.append("--removable")
        if self.clVars.Get('cl_action') != "system" and \
                not isMount('/boot/efi'):
            raise DistributiveError(_("Failed to install the bootloader. "
                                      "/boot/efi is not mounted."))
        grub_process = process(cmd_grub_install,
                               *grub_params, stderr=STDOUT,
                               envdict=os.environ)
        if grub_process.failed():
            raise DistributiveError(_("Failed to install the bootloader"))
        # проверяем успешность создания загрузочной записи
        # если среди загрузочных записей отсутствует запись
        # calculate и dmesg содержит сообщение об ошибке efivars -
        # запись создать не удалось
        efi_boot_mgr = getProgPath('/usr/sbin/efibootmgr')
        dmesg = getProgPath('/bin/dmesg')
        if efi_boot_mgr and dmesg:
            if not re.search('Boot.*calculate',
                             process(efi_boot_mgr).read(), re.M) and \
                    re.search('efivars.*set_variable.*failed',
                              process(dmesg).read(), re.M):
                raise DistributiveError(
                    _("Failed to create the UEFI boot record"))

    def installLegacyGrubBootloader(self, target):
        """
        Install legecy grub boot loader

        Perform grub installation to disk, which has root partition
        """
        cmd_grub = getProgPath('/sbin/grub')
        if not cmd_grub:
            raise DistributiveError(_("Failed to install the bootloader"))
        grub_process = process(
            cmd_grub,
            "--device-map=%s/boot/grub/device.map" % target.getDirectory(),
            "--batch", stderr=STDOUT)
        boot_disk = self.clVars.Select('os_install_disk_grub',
                                       where='os_install_disk_mount',
                                       _in=('/', '/boot'),
                                       sort='DESC', limit=1)
        if not boot_disk:
            raise DistributiveError(_("Failed to determine the boot disk"))
        self.setActivePartition(boot_disk)
        for mbr_disk in self.clVars.Get('os_install_mbr'):
            mbr_disk_num = self.clVars.Select("os_device_map",
                                              where="os_device_dev",
                                              eq=mbr_disk)
            if not mbr_disk_num and mbr_disk_num != 0:
                raise DistributiveError(_("Failed to determine mbr"))
            for line in ("root (hd%s)" % boot_disk,
                         "setup (hd%d)" % mbr_disk_num,
                         "quit"):
                grub_process.write("%s\n" % line)
            if grub_process.failed():
                raise DistributiveError(_("Failed to install the bootloader"))

    def setupOpenGL(self):
        """
        Выполнить выбор opengl для текущего видеодрайвера
        """
        default_gl = "xorg-x11"
        path_gl_modules = path.join(self.clVars.Get('cl_chroot_path'),
                                    'usr/lib/opengl')
        open_gl_env = path.join(self.clVars.Get('cl_chroot_path'),
                                'etc/env.d/03opengl')

        open_gl_mods = filter(lambda x: x != "global",
                              listDirectory(path_gl_modules))
        map_gl_drivers = {'fglrx': ("ati" if "ati" in open_gl_mods
                                    else default_gl),
                          'nvidia': "nvidia" if "nvidia" in open_gl_mods
                          else default_gl}
        x11_driver = self.clVars.Get('os_install_x11_video_drv')
        if x11_driver in map_gl_drivers:
            new_module_name = map_gl_drivers[x11_driver]
        else:
            new_module_name = default_gl
        current_module_name = map(
            lambda x: x.strip().rpartition('=')[-1].strip('"\''),
            filter(lambda x: x.startswith("OPENGL_PROFILE="),
                   readLinesFile(open_gl_env)))
        if current_module_name:
            current_module_name = current_module_name[-1]
        else:
            current_module_name = ""
        if current_module_name == new_module_name:
            return True
        return process('/usr/bin/eselect', 'opengl', 'set',
                       new_module_name).success()

    def checkVideoDriver(self):
        """
        Проверить видео драйвер, и если это nvidia, то
        обновить маску на пакет видеодрайвера
        """
        if self.clVars.Get('hr_video') != 'nvidia':
            return True
        mask_file = '/etc/portage/package.mask'
        nvidia_mask_file = path.join(mask_file, 'nvidia')
        # если package.mask является файлом - делаем его директорией
        if path.isfile(mask_file):
            os.rename(mask_file, mask_file + "2")
            os.mkdir(mask_file, mode=0755)
            os.rename(mask_file + "2", path.join(mask_file, "default"))
        current_nvidia_mask = readFile(nvidia_mask_file).strip()
        new_nvidia_mask = self.clVars.Get('os_nvidia_mask')
        if new_nvidia_mask == current_nvidia_mask:
            return True
        open(nvidia_mask_file, 'w').write(new_nvidia_mask)
        return True

    def changeScheduler(self, scheduler):
        """
        Изменить текущий IO планировщик
        """
        root_dev = self.clVars.Select('os_disk_parent',
                                      where='os_disk_mount',
                                      eq='/', limit=1)
        try:
            scheduler_path = (
                "/sys%s/queue/scheduler" %
                (getUdevDeviceInfo(name=root_dev).get('DEVPATH', '')))
            if path.exists(scheduler_path):
                open(scheduler_path, 'w').write(scheduler)
        except Exception:
            raise InstallError(_("Unable to change the I/O scheduler"))
        return True

    def autopartition(self, table, devices, data, lvm, lvm_vgname, bios_grub,
                      bios_grub_size):
        """
        Авторазметка диска с таблицей разделов 'table', диски указываются
        'device', параметры таблицы 'data', 'lvm' использование LVM,
        'lvm_vgname' название группы томов LVM, bios_grub - создавать
        bios_grub раздел, bios_grub_size - раздел bios grub раздела в байтах
        """
        ap = AutoPartition()
        ap.clearLvm(devices, self.clVars)
        ap.clearRaid(devices, self.clVars)
        ap.recreateSpace(table, devices, data, lvm,
                         lvm_vgname, bios_grub, bios_grub_size)
        return True

    def format(self, target):
        """
        Форматировать разделы для 'target' дистрибутива
        """
        target.performFormat()
        return True

    def unpack(self, source, target, files_num):
        """
        Распаковать 'source' в 'target', 'filesnum' количество копируемых файлов
        """
        self.addProgress()
        if files_num.isdigit():
            files_num = int(files_num)
        else:
            files_num = 0
        target.installFrom(source, callbackProgress=self.setProgress,
                           filesnum=files_num)
        return True

    def copyClt(self, source, target, cltpath):
        """
        Скопировать clt шаблоны из 'cltpath' в 'target' дистрибутив из
        'source' дистрибутива
        """
        target_dir = target.getDirectory()
        source_dir = source.getDirectory()
        for f in filter(lambda x: x.endswith('.clt'),
                        chain(*map(lambda x: find(pathJoin(source_dir, x),
                                                  filetype="f"),
                                   cltpath))):
            copyWithPath(f, target_dir, prefix=source_dir)
        return True

    def copyOther(self, source, target):
        """
        Скопировать прочие настройки из текущей системы в новую
        """
        file_mask = re.compile("(/etc/ssh/ssh_host_.*|"
                               "/root/.ssh/(id_.*|known_hosts))")
        target_dir = target.getDirectory()
        source_dir = source.getDirectory()
        for f in filter(file_mask.search,
                        chain(*map(lambda x: find(pathJoin(source_dir, x),
                                                  filetype="f"),
                                   ["/etc", "/root/.ssh"]))):
            copyWithPath(f, target_dir, prefix=source_dir)
        return True

    def rndString(self):
        """
        Получить произвольную строку из 8 символов
        """
        """Get random string with len 8 char"""
        return "".join([choice(string.ascii_letters + string.digits)
                        for i in xrange(0, 8)])

    def _getFreeDirectory(self, directory):
        """
        Получить название директории
        """
        new_dir_name = directory
        while path.exists(new_dir_name):
            new_dir_name = "%s.%s" % (directory, self.rndString())
        return new_dir_name

    def remountNTFS(self):
        """
        Перемонтировать NTFS разделы для работы os-prober
        """
        res = True
        for disk in self.clVars.Select('os_disk_dev',
                                       where='os_disk_format', like='ntfs'):
            mount_dir = self._getFreeDirectory('/var/lib/calculate/mount.ntfs')
            try:
                os.mkdir(mount_dir)
            except (OSError, IOError):
                continue
            if process('/bin/mount', disk, mount_dir).success():
                for i in (0.2, 0.5, 1, 2, 4, 5):
                    if process('/bin/umount', mount_dir).success():
                        break
                    time.sleep(i)
                else:
                    self.printWARNING(_("Unable to umount %s") % mount_dir)
                    res = False
            try:
                os.rmdir(mount_dir)
            except (OSError, IOError):
                self.printWARNING(
                    _("Unable to remove directory %s") % mount_dir)
                return False
        return res

    def mountBind(self, target):
        """
        Подключить bind точки монтирования у дистрибутива
        """
        target.postinstallMountBind()
        return True

    def userMigrate(self, target, migrate_data, root_pwd):
        """
        Перенос текущих пользователей в новую систему,
        установка пароля пользователя root
        """
        migrator = migrate(target.getDirectory())
        if not migrator.migrate(migrate_data, root_pwd, [], [], ):
            raise InstallError(_("Failed to migrate users onto the new system"))
        return True

    def umount(self, distr):
        """
        Отключить дистрибутив
        """
        distr.close()
        return True

    def drop_xorg_logs(self):
        """
        Сбросить логи загрузки xorg сервера
        """
        for fn in glob.glob("/var/log/Xorg.*.log"):
            new_name = "%s.old" % fn
            if path.exists(new_name):
                os.unlink(new_name)
            shutil.move(fn, new_name)
        return True